def translate(self, dr):
        """
        Translate a tile by an amount dr

        >>> Tile(5).translate(1)
        Tile [1, 1, 1] -> [6, 6, 6] ([5, 5, 5])
        """
        tile = self.copy()
        tile.l += dr
        tile.r += dr
        return tile

def translate ( self , dr ) : tile = self . copy ( ) tile . l += dr tile . r += dr return tile


def pad(self, pad):
        """
        Pad this tile by an equal amount on each side as specified by pad

        >>> Tile(10).pad(2)
        Tile [-2, -2, -2] -> [12, 12, 12] ([14, 14, 14])

        >>> Tile(10).pad([1,2,3])
        Tile [-1, -2, -3] -> [11, 12, 13] ([12, 14, 16])
        """
        tile = self.copy()
        tile.l -= pad
        tile.r += pad
        return tile

def pad ( self , pad ) : tile = self . copy ( ) tile . l -= pad tile . r += pad return tile


def filtered_image(self, im):
        """Returns a filtered image after applying the Fourier-space filters"""
        q = np.fft.fftn(im)
        for k,v in self.filters:
            q[k] -= v
        return np.real(np.fft.ifftn(q))

def filtered_image ( self , im ) : q = np . fft . fftn ( im ) for k , v in self . filters : q [ k ] -= v return np . real ( np . fft . ifftn ( q ) )

Returns a filtered image after applying the Fourier - space filters


def set_filter(self, slices, values):
        """
        Sets Fourier-space filters for the image. The image is filtered by
        subtracting values from the image at slices.

        Parameters
        ----------
        slices : List of indices or slice objects.
            The q-values in Fourier space to filter.
        values : np.ndarray
            The complete array of Fourier space peaks to subtract off.  values
            should be the same size as the FFT of the image; only the portions
            of values at slices will be removed.

        Examples
        --------
        To remove a two Fourier peaks in the data at q=(10, 10, 10) &
        (245, 245, 245), where im is the residuals of a model:

            * slices = [(10,10,10), (245, 245, 245)]
            * values = np.fft.fftn(im)
            * im.set_filter(slices, values)
        """
        self.filters = [[sl,values[sl]] for sl in slices]

def set_filter ( self , slices , values ) : self . filters = [ [ sl , values [ sl ] ] for sl in slices ]

Sets Fourier - space filters for the image . The image is filtered by subtracting values from the image at slices .


def load_image(self):
        """ Read the file and perform any transforms to get a loaded image """
        try:
            image = initializers.load_tiff(self.filename)
            image = initializers.normalize(
                image, invert=self.invert, scale=self.exposure,
                dtype=self.float_precision
            )
        except IOError as e:
            log.error("Could not find image '%s'" % self.filename)
            raise e

        return image

def
load_image
(
self
)
:
try
:
image
=
initializers
.
load_tiff
(
self
.
filename
)
image
=
initializers
.
normalize
(
image
,
invert
=
self
.
invert
,
scale
=
self
.
exposure
,
dtype
=
self
.
float_precision
)
except
IOError
as
e
:
log
.
error
(
"Could not find image '%s'"
%
self
.
filename
)
raise
e
return
image


def get_scale_from_raw(raw, scaled):
        """
        When given a raw image and the scaled version of the same image, it
        extracts the ``exposure`` parameters associated with those images.
        This is useful when

        Parameters
        ----------
        raw : array_like
            The image loaded fresh from a file

        scaled : array_like
            Image scaled using :func:`peri.initializers.normalize`

        Returns
        -------
        exposure : tuple of numbers
            Returns the exposure parameters (emin, emax) which get mapped to
            (0, 1) in the scaled image. Can be passed to
            :func:`~peri.util.RawImage.__init__`
        """
        t0, t1 = scaled.min(), scaled.max()
        r0, r1 = float(raw.min()), float(raw.max())

        rmin = (t1*r0 - t0*r1) / (t1 - t0)
        rmax = (r1 - r0) / (t1 - t0) + rmin
        return (rmin, rmax)

def get_scale_from_raw ( raw , scaled ) : t0 , t1 = scaled . min ( ) , scaled . max ( ) r0 , r1 = float ( raw . min ( ) ) , float ( raw . max ( ) ) rmin = ( t1 * r0 - t0 * r1 ) / ( t1 - t0 ) rmax = ( r1 - r0 ) / ( t1 - t0 ) + rmin return ( rmin , rmax )

When given a raw image and the scaled version of the same image it extracts the exposure parameters associated with those images . This is useful when


def _draw(self):
        """ Interal draw method, simply prints to screen """
        if self.display:
            print(self._formatstr.format(**self.__dict__), end='')
            sys.stdout.flush()

def _draw ( self ) : if self . display : print ( self . _formatstr . format ( * * self . __dict__ ) , end = '' ) sys . stdout . flush ( )

Interal draw method simply prints to screen


def update(self, value=0):
        """
        Update the value of the progress and update progress bar.

        Parameters
        -----------
        value : integer
            The current iteration of the progress
        """
        self._deltas.append(time.time())

        self.value = value
        self._percent = 100.0 * self.value / self.num

        if self.bar:
            self._bars = self._bar_symbol*int(np.round(self._percent / 100. * self._barsize))

        if (len(self._deltas) < 2) or (self._deltas[-1] - self._deltas[-2]) > 1e-1:
            self._estimate_time()
            self._draw()

        if self.value == self.num:
            self.end()

def update ( self , value = 0 ) : self . _deltas . append ( time . time ( ) ) self . value = value self . _percent = 100.0 * self . value / self . num if self . bar : self . _bars = self . _bar_symbol * int ( np . round ( self . _percent / 100. * self . _barsize ) ) if ( len ( self . _deltas ) < 2 ) or ( self . _deltas [ - 1 ] - self . _deltas [ - 2 ] ) > 1e-1 : self . _estimate_time ( ) self . _draw ( ) if self . value == self . num : self . end ( )

Update the value of the progress and update progress bar .


def check_consistency(self):
        """
        Make sure that the required comps are included in the list of
        components supplied by the user. Also check that the parameters are
        consistent across the many components.
        """
        error = False
        regex = re.compile('([a-zA-Z_][a-zA-Z0-9_]*)')

        # there at least must be the full model, not necessarily partial updates
        if 'full' not in self.modelstr:
            raise ModelError(
                'Model must contain a `full` key describing '
                'the entire image formation'
            )

        # Check that the two model descriptors are consistent
        for name, eq in iteritems(self.modelstr):
            var = regex.findall(eq)
            for v in var:
                # remove the derivative signs if there (dP -> P)
                v = re.sub(r"^d", '', v)
                if v not in self.varmap:
                    log.error(
                        "Variable '%s' (eq. '%s': '%s') not found in category map %r" %
                        (v, name, eq, self.varmap)
                    )
                    error = True

        if error:
            raise ModelError('Inconsistent varmap and modelstr descriptions')

Make sure that the required comps are included in the list of components supplied by the user . Also check that the parameters are consistent across the many components .


def check_inputs(self, comps):
        """
        Check that the list of components `comp` is compatible with both the
        varmap and modelstr for this Model
        """
        error = False
        compcats = [c.category for c in comps]

        # Check that the components are all provided, given the categories
        for k, v in iteritems(self.varmap):
            if k not in self.modelstr['full']:
                log.warn('Component (%s : %s) not used in model.' % (k,v))

            if v not in compcats:
                log.error('Map component (%s : %s) not found in list of components.' % (k,v))
                error = True

        if error:
            raise ModelError('Component list incomplete or incorrect')

Check that the list of components comp is compatible with both the varmap and modelstr for this Model


def lbl(axis, label, size=22):
    """ Put a figure label in an axis """
    at = AnchoredText(label, loc=2, prop=dict(size=size), frameon=True)
    at.patch.set_boxstyle("round,pad=0.,rounding_size=0.0")
    #bb = axis.get_yaxis_transform()
    #at = AnchoredText(label,
    #        loc=3, prop=dict(size=18), frameon=True,
    #        bbox_to_anchor=(-0.5,1),#(-.255, 0.90),
    #        bbox_transform=bb,#axis.transAxes
    #    )
    axis.add_artist(at)

#        bbox_transform=bb,#axis.transAxes


def examine_unexplained_noise(state, bins=1000, xlim=(-10,10)):
    """
    Compares a state's residuals in real and Fourier space with a Gaussian.

    Point out that Fourier space should always be Gaussian and white

    Parameters
    ----------
        state : `peri.states.State`
            The state to examine.
        bins : int or sequence of scalars or str, optional
            The number of bins in the histogram, as passed to numpy.histogram
            Default is 1000
        xlim : 2-element tuple, optional
            The range, in sigma, of the x-axis on the plot. Default (-10,10).

    Returns
    -------
        list
            The axes handles for the real and Fourier space subplots.
    """
    r = state.residuals
    q = np.fft.fftn(r)
    #Get the expected values of `sigma`:
    calc_sig = lambda x: np.sqrt(np.dot(x,x) / x.size)
    rh, xr = np.histogram(r.ravel() / calc_sig(r.ravel()), bins=bins,
            density=True)
    bigq = np.append(q.real.ravel(), q.imag.ravel())
    qh, xq = np.histogram(bigq / calc_sig(q.real.ravel()), bins=bins,
            density=True)
    xr = 0.5*(xr[1:] + xr[:-1])
    xq = 0.5*(xq[1:] + xq[:-1])

    gauss = lambda t : np.exp(-t*t*0.5) / np.sqrt(2*np.pi)

    plt.figure(figsize=[16,8])
    axes = []
    for a, (x, r, lbl) in enumerate([[xr, rh, 'Real'], [xq, qh, 'Fourier']]):
        ax = plt.subplot(1,2,a+1)
        ax.semilogy(x, r, label='Data')
        ax.plot(x, gauss(x), label='Gauss Fit', scalex=False, scaley=False)
        ax.set_xlabel('Residuals value $r/\sigma$')
        ax.set_ylabel('Probability $P(r/\sigma)$')
        ax.legend(loc='upper right')
        ax.set_title('{}-Space'.format(lbl))
        ax.set_xlim(xlim)
        axes.append(ax)
    return axes

def
examine_unexplained_noise
(
state
,
bins
=
1000
,
xlim
=
(
10
,
10
)
)
:
r
=
state
.
residuals
q
=
np
.
fft
.
fftn
(
r
)
#Get the expected values of `sigma`:
calc_sig
=
lambda
x
:
np
.
sqrt
(
np
.
dot
(
x
,
x
)
/
x
.
size
)
rh
,
xr
=
np
.
histogram
(
r
.
ravel
(
)
/
calc_sig
(
r
.
ravel
(
)
)
,
bins
=
bins
,
density
=
True
)
bigq
=
np
.
append
(
q
.
real
.
ravel
(
)
,
q
.
imag
.
ravel
(
)
)
qh
,
xq
=
np
.
histogram
(
bigq
/
calc_sig
(
q
.
real
.
ravel
(
)
)
,
bins
=
bins
,
density
=
True
)
xr
=
0.5
*
(
xr
[
1
:
]
+
xr
[
:
1
]
)
xq
=
0.5
*
(
xq
[
1
:
]
+
xq
[
:
1
]
)
gauss
=
lambda
t
:
np
.
exp
(
t
*
t
*
0.5
)
/
np
.
sqrt
(
2
*
np
.
pi
)
plt
.
figure
(
figsize
=
[
16
,
8
]
)
axes
=
[
]
for
a
,
(
x
,
r
,
lbl
)
in
enumerate
(
[
[
xr
,
rh
,
'Real'
]
,
[
xq
,
qh
,
'Fourier'
]
]
)
:
ax
=
plt
.
subplot
(
1
,
2
,
a
+
1
)
ax
.
semilogy
(
x
,
r
,
label
=
'Data'
)
ax
.
plot
(
x
,
gauss
(
x
)
,
label
=
'Gauss Fit'
,
scalex
=
False
,
scaley
=
False
)
ax
.
set_xlabel
(
'Residuals value $r/\sigma$'
)
ax
.
set_ylabel
(
'Probability $P(r/\sigma)$'
)
ax
.
legend
(
loc
=
'upper right'
)
ax
.
set_title
(
'{}-Space'
.
format
(
lbl
)
)
ax
.
set_xlim
(
xlim
)
axes
.
append
(
ax
)
return
axes

Compares a state s residuals in real and Fourier space with a Gaussian .


def compare_data_model_residuals(s, tile, data_vmin='calc', data_vmax='calc',
         res_vmin=-0.1, res_vmax=0.1, edgepts='calc', do_imshow=True,
         data_cmap=plt.cm.bone, res_cmap=plt.cm.RdBu):
    """
    Compare the data, model, and residuals of a state.

    Makes an image of any 2D slice of a state that compares the data,
    model, and residuals. The upper left portion of the image is the raw
    data, the central portion the model, and the lower right portion the
    image. Either plots the image using plt.imshow() or returns a
    np.ndarray of the image pixels for later use.

    Parameters
    ----------
        st : peri.ImageState object
            The state to plot.
        tile : peri.util.Tile object
            The slice of the image to plot. Can be any xy, xz, or yz
            projection, but it must return a valid 2D slice (the slice is
            squeezed internally).

        data_vmin : {Float, `calc`}, optional
            vmin for the imshow for the data and generative model (shared).
            Default is 'calc' = 0.5(data.min() + model.min())
        data_vmax : {Float, `calc`}, optional
            vmax for the imshow for the data and generative model (shared).
            Default is 'calc' = 0.5(data.max() + model.max())
        res_vmin : Float, optional
            vmin for the imshow for the residuals. Default is -0.1
            Default is 'calc' = 0.5(data.min() + model.min())
        res_vmax : Float, optional
            vmax for the imshow for the residuals. Default is +0.1
        edgepts : {Nested list-like, Float, 'calc'}, optional.
            The vertices of the triangles which determine the splitting of
            the image. The vertices are at (image corner, (edge, y), and
            (x,edge), where edge is the appropriate edge of the image.
                edgepts[0] : (x,y) points for the upper edge
                edgepts[1] : (x,y) points for the lower edge
            where `x` is the coordinate along the image's 0th axis and `y`
            along the images 1st axis. Default is 'calc,' which calculates
            edge points by splitting the image into 3 regions of equal
            area. If edgepts is a float scalar, calculates the edge points
            based on a constant fraction of distance from the edge.
        do_imshow : Bool
            If True, imshow's and returns the returned handle.
            If False, returns the array as a [M,N,4] array.
        data_cmap : matplotlib colormap instance
            The colormap to use for the data and model.
        res_cmap : matplotlib colormap instance
            The colormap to use for the residuals.

    Returns
    -------
        image : {matplotlib.pyplot.AxesImage, numpy.ndarray}
            If `do_imshow` == True, the returned handle from imshow.
            If `do_imshow` == False, an [M,N,4] np.ndarray of the image
            pixels.
    """
    # This could be modified to alpha the borderline... or to embiggen
    # the image and slice it more finely
    residuals = s.residuals[tile.slicer].squeeze()
    data = s.data[tile.slicer].squeeze()
    model = s.model[tile.slicer].squeeze()
    if data.ndim != 2:
        raise ValueError('tile does not give a 2D slice')

    im = np.zeros([data.shape[0], data.shape[1], 4])
    if data_vmin == 'calc':
        data_vmin = 0.5*(data.min() + model.min())
    if data_vmax == 'calc':
        data_vmax = 0.5*(data.max() + model.max())

    #1. Get masks:
    upper_mask, center_mask, lower_mask = trisect_image(im.shape, edgepts)

    #2. Get colorbar'd images
    gm = data_cmap(center_data(model, data_vmin, data_vmax))
    dt = data_cmap(center_data(data, data_vmin, data_vmax))
    rs = res_cmap(center_data(residuals, res_vmin, res_vmax))

    for a in range(4):
        im[:,:,a][upper_mask] = rs[:,:,a][upper_mask]
        im[:,:,a][center_mask] = gm[:,:,a][center_mask]
        im[:,:,a][lower_mask] = dt[:,:,a][lower_mask]
    if do_imshow:
        return plt.imshow(im)
    else:
        return im

Compare the data model and residuals of a state .


def trisect_image(imshape, edgepts='calc'):
    """
    Returns 3 masks that trisect an image into 3 triangular portions.

    Parameters
    ----------
        imshape : 2-element list-like of ints
            The shape of the image. Elements after the first 2 are ignored.

        edgepts : Nested list-like, float, or `calc`, optional.
            The vertices of the triangles which determine the splitting of
            the image. The vertices are at (image corner, (edge, y), and
            (x,edge), where edge is the appropriate edge of the image.
                edgepts[0] : (x,y) points for the upper edge
                edgepts[1] : (x,y) points for the lower edge
            where `x` is the coordinate along the image's 0th axis and `y`
            along the images 1st axis. Default is 'calc,' which calculates
            edge points by splitting the image into 3 regions of equal
            area. If edgepts is a float scalar, calculates the edge points
            based on a constant fraction of distance from the edge.

    Returns
    -------
        upper_mask : numpy.ndarray
            Boolean array; True in the image's upper  region.
        center_mask : numpy.ndarray
            Boolean array; True in the image's center region.
        lower_mask : numpy.ndarray
            Boolean array; True in the image's lower  region.
    """
    im_x, im_y = np.meshgrid(np.arange(imshape[0]), np.arange(imshape[1]),
            indexing='ij')
    if np.size(edgepts) == 1:
        #Gets equal-area sections, at sqrt(2/3) of the sides
        f = np.sqrt(2./3.) if edgepts == 'calc' else edgepts
        # f = np.sqrt(2./3.)
        lower_edge = (imshape[0] * (1-f),  imshape[1] * f)
        upper_edge = (imshape[0] * f,      imshape[1] * (1-f))
    else:
        upper_edge, lower_edge = edgepts

    #1. Get masks
    lower_slope = lower_edge[1] / max(float(imshape[0] - lower_edge[0]), 1e-9)
    upper_slope = (imshape[1] - upper_edge[1]) / float(upper_edge[0])
    #and the edge points are the x or y intercepts
    lower_intercept = -lower_slope * lower_edge[0]
    upper_intercept = upper_edge[1]
    lower_mask = im_y < (im_x * lower_slope + lower_intercept)
    upper_mask = im_y > (im_x * upper_slope + upper_intercept)

    center_mask= -(lower_mask | upper_mask)
    return upper_mask, center_mask, lower_mask

Returns 3 masks that trisect an image into 3 triangular portions .


def sim_crb_diff(std0, std1, N=10000):
    """ each element of std0 should correspond with the element of std1 """
    a = std0*np.random.randn(N, len(std0))
    b = std1*np.random.randn(N, len(std1))
    return a - b

def sim_crb_diff ( std0 , std1 , N = 10000 ) : a = std0 * np . random . randn ( N , len ( std0 ) ) b = std1 * np . random . randn ( N , len ( std1 ) ) return a - b


def twoslice(field, center=None, size=6.0, cmap='bone_r', vmin=0, vmax=1,
        orientation='vertical', figpad=1.09, off=0.01):
    """
    Plot two parts of the ortho view, the two sections given by ``orientation``.
    """
    center = center or [i//2 for i in field.shape]
    slices = []
    for i,c in enumerate(center):
        blank = [np.s_[:]]*len(center)
        blank[i] = c
        slices.append(tuple(blank))

    z,y,x = [float(i) for i in field.shape]
    w = float(x + z)
    h = float(y + z)

    def show(field, ax, slicer, transpose=False):
        tmp = field[slicer] if not transpose else field[slicer].T
        ax.imshow(
            tmp, cmap=cmap, interpolation='nearest',
            vmin=vmin, vmax=vmax
        )
        ax.set_xticks([])
        ax.set_yticks([])
        ax.grid('off')

    if orientation.startswith('v'):
        # rect = l,b,w,h
        log.info('{} {} {} {} {} {}'.format(x, y, z, w, h, x/h))
        r = x/h
        q = y/h
        f = 1 / (1 + 3*off)
        fig = pl.figure(figsize=(size*r, size*f))
        ax1 = fig.add_axes((off, f*(1-q)+2*off, f, f*q))
        ax2 = fig.add_axes((off, off,           f, f*(1-q)))

        show(field, ax1, slices[0])
        show(field, ax2, slices[1])
    else:
        # rect = l,b,w,h
        r = y/w
        q = x/w
        f = 1 / (1 + 3*off)
        fig = pl.figure(figsize=(size*f, size*r))
        ax1 = fig.add_axes((off,    off,   f*q, f))
        ax2 = fig.add_axes((2*off+f*q, off, f*(1-q), f))

        show(field, ax1, slices[0])
        show(field, ax2, slices[2], transpose=True)

    return fig, ax1, ax2

def
twoslice
(
field
,
center
=
,
size
=
6.0
,
cmap
=
'bone_r'
,
vmin
=
0
,
vmax
=
1
,
orientation
=
'vertical'
,
figpad
=
1.09
,
off
=
0.01
)
:
center
=
center
or
[
i
//
2
for
i
in
field
.
shape
]
slices
=
[
]
for
i
,
c
in
enumerate
(
center
)
:
blank
=
[
np
.
s_
[
:
]
]
*
len
(
center
)
blank
[
i
]
=
c
slices
.
append
(
tuple
(
blank
)
)
z
,
y
,
x
=
[
float
(
i
)
for
i
in
field
.
shape
]
w
=
float
(
x
+
z
)
h
=
float
(
y
+
z
)
def
show
(
field
,
ax
,
slicer
,
transpose
=
False
)
:
tmp
=
field
[
slicer
]
if
not
transpose
else
field
[
slicer
]
.
T
ax
.
imshow
(
tmp
,
cmap
=
cmap
,
interpolation
=
'nearest'
,
vmin
=
vmin
,
vmax
=
vmax
)
ax
.
set_xticks
(
[
]
)
ax
.
set_yticks
(
[
]
)
ax
.
grid
(
'off'
)
if
orientation
.
startswith
(
'v'
)
:
# rect = l,b,w,h
log
.
info
(
'{} {} {} {} {} {}'
.
format
(
x
,
y
,
z
,
w
,
h
,
x
/
h
)
)
r
=
x
/
h
q
=
y
/
h
f
=
1
/
(
1
+
3
*
off
)
fig
=
pl
.
figure
(
figsize
=
(
size
*
r
,
size
*
f
)
)
ax1
=
fig
.
add_axes
(
(
off
,
f
*
(
1
q
)
+
2
*
off
,
f
,
f
*
q
)
)
ax2
=
fig
.
add_axes
(
(
off
,
off
,
f
,
f
*
(
1
q
)
)
)
show
(
field
,
ax1
,
slices
[
0
]
)
show
(
field
,
ax2
,
slices
[
1
]
)
else
:
# rect = l,b,w,h
r
=
y
/
w
q
=
x
/
w
f
=
1
/
(
1
+
3
*
off
)
fig
=
pl
.
figure
(
figsize
=
(
size
*
f
,
size
*
r
)
)
ax1
=
fig
.
add_axes
(
(
off
,
off
,
f
*
q
,
f
)
)
ax2
=
fig
.
add_axes
(
(
2
*
off
+
f
*
q
,
off
,
f
*
(
1
q
)
,
f
)
)
show
(
field
,
ax1
,
slices
[
0
]
)
show
(
field
,
ax2
,
slices
[
2
]
,
transpose
=
True
)
return
fig
,
ax1
,
ax2

Plot two parts of the ortho view the two sections given by orientation .


def circles(st, layer, axis, ax=None, talpha=1.0, cedge='white', cface='white'):
    """
    Plots a set of circles corresponding to a slice through the platonic
    structure. Copied from twoslice_overlay with comments, standaloneness.

    Inputs
    ------
        pos : array of particle positions; [N,3]
        rad : array of particle radii; [N]
        ax : plt.axis instance
        layer : Which layer of the slice to use.
        axis : The slice of the image, 0, 1, or 2.
        cedge : edge color
        cface : face color
        talpha : Alpha of the thing
    """
    pos = st.obj_get_positions()
    rad = st.obj_get_radii()
    shape = st.ishape.shape.tolist()
    shape.pop(axis) #shape is now the shape of the image
    if ax is None:
        fig = plt.figure()
        axisbg = 'white' if cface == 'black' else 'black'
        sx, sy = ((1,shape[1]/float(shape[0])) if shape[0] > shape[1] else
                (shape[0]/float(shape[1]), 1))
        ax = fig.add_axes((0,0, sx, sy), axisbg=axisbg)
    # get the index of the particles we want to include
    particles = np.arange(len(pos))[np.abs(pos[:,axis] - layer) < rad]

    # for each of these particles display the effective radius
    # in the proper place
    scale = 1.0 #np.max(shape).astype('float')
    for i in particles:
        p = pos[i].copy()
        r = 2*np.sqrt(rad[i]**2 - (p[axis] - layer)**2)
        #CIRCLE IS IN FIGURE COORDINATES!!!
        if axis==0:
            ix = 1; iy = 2
        elif axis == 1:
            ix = 0; iy = 2
        elif axis==2:
            ix = 0; iy = 1
        c = Circle((p[ix]/scale, p[iy]/scale), radius=r/2/scale, fc=cface,
                ec=cedge, alpha=talpha)
        ax.add_patch(c)
    # plt.axis([0,1,0,1])
    plt.axis('equal') #circles not ellipses
    return ax

Plots a set of circles corresponding to a slice through the platonic structure . Copied from twoslice_overlay with comments standaloneness .


def missing_particle(separation=0.0, radius=RADIUS, SNR=20):
    """ create a two particle state and compare it to featuring using a single particle guess """
    # create a base image of one particle
    s = init.create_two_particle_state(imsize=6*radius+4, axis='x', sigma=1.0/SNR,
            delta=separation, radius=radius, stateargs={'varyn': True}, psfargs={'error': 1e-6})
    s.obj.typ[1] = 0.
    s.reset()

    return s, s.obj.pos.copy()

def
missing_particle
(
separation
=
0.0
,
radius
=
RADIUS
,
SNR
=
20
)
:
# create a base image of one particle
s
=
init
.
create_two_particle_state
(
imsize
=
6
*
radius
+
4
,
axis
=
'x'
,
sigma
=
1.0
/
SNR
,
delta
=
separation
,
radius
=
radius
,
stateargs
=
{
'varyn'
:
True
}
,
psfargs
=
{
'error'
:
1e-6
}
)
s
.
obj
.
typ
[
1
]
=
0.
s
.
reset
(
)
return
s
,
s
.
obj
.
pos
.
copy
(
)


def name_globals(s, remove_params=None):
    """
    Returns a list of the global parameter names.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to name the globals of.
        remove_params : Set or None
            A set of unique additional parameters to remove from the globals
            list.

    Returns
    -------
        all_params : list
            The list of the global parameter names, with each of
            remove_params removed.
    """
    all_params = s.params
    for p in s.param_particle(np.arange(s.obj_get_positions().shape[0])):
        all_params.remove(p)
    if remove_params is not None:
        for p in set(remove_params):
            all_params.remove(p)
    return all_params

def name_globals ( s , remove_params = None ) : all_params = s . params for p in s . param_particle ( np . arange ( s . obj_get_positions ( ) . shape [ 0 ] ) ) : all_params . remove ( p ) if remove_params is not None : for p in set ( remove_params ) : all_params . remove ( p ) return all_params

Returns a list of the global parameter names .


def get_num_px_jtj(s, nparams, decimate=1, max_mem=1e9, min_redundant=20):
    """
    Calculates the number of pixels to use for J at a given memory usage.

    Tries to pick a number of pixels as (size of image / `decimate`).
    However, clips this to a maximum size and minimum size to ensure that
    (1) too much memory isn't used and (2) J has enough elements so that
    the inverse of JTJ will be well-conditioned.

    Parameters
    ----------
        s : :class:`peri.states.State`
            The state on which to calculate J.
        nparams : Int
            The number of parameters that will be included in J.
        decimate : Int, optional
            The amount to decimate the number of pixels in the image by,
            i.e. tries to pick num_px = size of image / decimate.
            Default is 1
        max_mem : Numeric, optional
            The maximum allowed memory, in bytes, for J to occupy at
            double-precision. Default is 1e9.
        min_redundant : Int, optional
            The number of pixels must be at least `min_redundant` *
            `nparams`. If not, an error is raised. Default is 20

    Returns
    -------
        num_px : Int
            The number of pixels at which to calcualte J.
    """
    #1. Max for a given max_mem:
    px_mem = int(max_mem // 8 // nparams) #1 float = 8 bytes
    #2. num_pix for a given redundancy
    px_red = min_redundant*nparams
    #3. And # desired for decimation
    px_dec = s.residuals.size//decimate

    if px_red > px_mem:
        raise RuntimeError('Insufficient max_mem for desired redundancy.')
    num_px = np.clip(px_dec, px_red, px_mem).astype('int')
    return num_px

Calculates the number of pixels to use for J at a given memory usage .


def vectorize_damping(params, damping=1.0, increase_list=[['psf-', 1e4]]):
    """
    Returns a non-constant damping vector, allowing certain parameters to be
    more strongly damped than others.

    Parameters
    ----------
        params : List
            The list of parameter names, in order.
        damping : Float
            The default value of the damping.
        increase_list: List
            A nested 2-element list of the params to increase and their
            scale factors. All parameters containing the string
            increase_list[i][0] are increased by a factor increase_list[i][1].
    Returns
    -------
        damp_vec : np.ndarray
            The damping vector to use.
    """
    damp_vec = np.ones(len(params)) * damping
    for nm, fctr in increase_list:
        for a in range(damp_vec.size):
            if nm in params[a]:
                damp_vec[a] *= fctr
    return damp_vec

def vectorize_damping ( params , damping = 1.0 , increase_list = [ [ 'psf-' , 1e4 ] ] ) : damp_vec = np . ones ( len ( params ) ) * damping for nm , fctr in increase_list : for a in range ( damp_vec . size ) : if nm in params [ a ] : damp_vec [ a ] *= fctr return damp_vec

Returns a non - constant damping vector allowing certain parameters to be more strongly damped than others .


def find_particles_in_tile(positions, tile):
    """
    Finds the particles in a tile, as numpy.ndarray of ints.

    Parameters
    ----------
        positions : `numpy.ndarray`
            [N,3] array of the particle positions to check in the tile
        tile : :class:`peri.util.Tile` instance
            Tile of the region inside which to check for particles.

    Returns
    -------
        numpy.ndarray, int
            The indices of the particles in the tile.
    """
    bools = tile.contains(positions)
    return np.arange(bools.size)[bools]

def find_particles_in_tile ( positions , tile ) : bools = tile . contains ( positions ) return np . arange ( bools . size ) [ bools ]

Finds the particles in a tile as numpy . ndarray of ints .


def separate_particles_into_groups(s, region_size=40, bounds=None,
        doshift=False):
    """
    Separates particles into convenient groups for optimization.

    Given a state, returns a list of groups of particles. Each group of
    particles are located near each other in the image. Every particle
    located in the desired region is contained in exactly 1 group.

    Parameters
    ----------
    s : :class:`peri.states.ImageState`
        The peri state to find particles in.
    region_size : Int or 3-element list-like of ints, optional
        The size of the box. Groups particles into boxes of shape
        (region_size[0], region_size[1], region_size[2]). If region_size
        is a scalar, the box is a cube of length region_size.
        Default is 40.
    bounds : 2-element list-like of 3-element lists, optional
        The sub-region of the image over which to look for particles.
            bounds[0]: The lower-left  corner of the image region.
            bounds[1]: The upper-right corner of the image region.
        Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
        image size, i.e. the default places every particle in the image
        somewhere in the groups.
    doshift : {True, False, `'rand'`}, optional
        Whether or not to shift the tile boxes by half a region size, to
        prevent the same particles to be chosen every time. If `'rand'`,
        randomly chooses either True or False. Default is False

    Returns
    -------
    particle_groups : List
        Each element of particle_groups is an int numpy.ndarray of the
        group of nearby particles. Only contains groups with a nonzero
        number of particles, so the elements don't necessarily correspond
        to a given image region.
    """
    imtile = s.oshape.translate(-s.pad)
    bounding_tile = (imtile if bounds is None else Tile(bounds[0], bounds[1]))
    rs = (np.ones(bounding_tile.dim, dtype='int')*region_size if
            np.size(region_size) == 1 else np.array(region_size))

    n_translate = np.ceil(bounding_tile.shape.astype('float')/rs).astype('int')
    particle_groups = []
    tile = Tile(left=bounding_tile.l, right=bounding_tile.l + rs)
    if doshift == 'rand':
        doshift = np.random.choice([True, False])
    if doshift:
        shift = rs // 2
        n_translate += 1
    else:
        shift = 0
    deltas = np.meshgrid(*[np.arange(i) for i in n_translate])
    positions = s.obj_get_positions()
    if bounds is None:
        # FIXME this (deliberately) masks a problem where optimization
        # places particles outside the image. However, it ensures that
        # all particles are in at least one group when `bounds is None`,
        # which is the use case within opt. The 1e-3 is to ensure that
        # they are inside the box and not on the edge.
        positions = np.clip(positions, imtile.l+1e-3, imtile.r-1e-3)
    groups = list(map(lambda *args: find_particles_in_tile(positions,
            tile.translate( np.array(args) * rs - shift)), *[d.ravel()
            for d in deltas]))

    for i in range(len(groups)-1, -1, -1):
        if groups[i].size == 0:
            groups.pop(i)
    assert _check_groups(s, groups)
    return groups

Separates particles into convenient groups for optimization .


def _check_groups(s, groups):
    """Ensures that all particles are included in exactly 1 group"""
    ans = []
    for g in groups:
        ans.extend(g)
    if np.unique(ans).size != np.size(ans):
        return False
    elif np.unique(ans).size != s.obj_get_positions().shape[0]:
        return False
    else:
        return (np.arange(s.obj_get_radii().size) == np.sort(ans)).all()

def _check_groups ( s , groups ) : ans = [ ] for g in groups : ans . extend ( g ) if np . unique ( ans ) . size != np . size ( ans ) : return False elif np . unique ( ans ) . size != s . obj_get_positions ( ) . shape [ 0 ] : return False else : return ( np . arange ( s . obj_get_radii ( ) . size ) == np . sort ( ans ) ) . all ( )


def calc_particle_group_region_size(s, region_size=40, max_mem=1e9, **kwargs):
    """
    Finds the biggest region size for LM particle optimization with a
    given memory constraint.

    Input Parameters
    ----------------
        s : :class:`peri.states.ImageState`
            The state with the particles
        region_size : Int or 3-element list-like of ints, optional.
            The initial guess for the region size. Default is 40
        max_mem : Numeric, optional
            The maximum memory for the optimizer to take. Default is 1e9

    Other Parameters
    ----------------
        bounds: 2-element list-like of 3-element lists.
            The sub-region of the image over which to look for particles.
                bounds[0]: The lower-left  corner of the image region.
                bounds[1]: The upper-right corner of the image region.
            Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
            image size, i.e. the default places every particle in the image
            somewhere in the groups.
    Returns
    -------
        region_size : numpy.ndarray of ints of the region size.
    """
    region_size = np.array(region_size).astype('int')

    def calc_mem_usage(region_size):
        rs = np.array(region_size)
        particle_groups = separate_particles_into_groups(s, region_size=
                rs.tolist(), **kwargs)
        # The actual mem usage is the max of the memory usage of all the
        # particle groups. However this is too slow. So instead we use the
        # max of the memory of the biggest 5 particle groups:
        numpart = [np.size(g) for g in particle_groups]
        biggroups = [particle_groups[i] for i in np.argsort(numpart)[-5:]]
        def get_tile_jsize(group):
            nms = s.param_particle(group)
            tile = s.get_update_io_tiles(nms, s.get_values(nms))[2]
            return tile.shape.prod() * len(nms)
        mems = [8*get_tile_jsize(g) for g in biggroups]  # 8 for bytes/float64
        return np.max(mems)

    im_shape = s.oshape.shape
    if calc_mem_usage(region_size) > max_mem:
        while ((calc_mem_usage(region_size) > max_mem) and
                np.any(region_size > 2)):
            region_size = np.clip(region_size-1, 2, im_shape)
    else:
        while ((calc_mem_usage(region_size) < max_mem) and
                np.any(region_size < im_shape)):
            region_size = np.clip(region_size+1, 2, im_shape)
        region_size -= 1 #need to be < memory, so we undo 1 iteration

    return region_size

Finds the biggest region size for LM particle optimization with a given memory constraint .


def get_residuals_update_tile(st, padded_tile):
    """
    Translates a tile in the padded image to the unpadded image.

    Given a state and a tile that corresponds to the padded image, returns
    a tile that corresponds to the the corresponding pixels of the difference
    image

    Parameters
    ----------
        st : :class:`peri.states.State`
            The state
        padded_tile : :class:`peri.util.Tile`
            The tile in the padded image.

    Returns
    -------
        :class:`peri.util.Tile`
            The tile corresponding to padded_tile in the unpadded image.
    """
    inner_tile = st.ishape.intersection([st.ishape, padded_tile])
    return inner_tile.translate(-st.pad)

def get_residuals_update_tile ( st , padded_tile ) : inner_tile = st . ishape . intersection ( [ st . ishape , padded_tile ] ) return inner_tile . translate ( - st . pad )

Translates a tile in the padded image to the unpadded image .


def find_best_step(err_vals):
    """
    Returns the index of the lowest of the passed values. Catches nans etc.
    """
    if np.all(np.isnan(err_vals)):
        raise ValueError('All err_vals are nans!')
    return np.nanargmin(err_vals)

def
find_best_step
(
err_vals
)
:
if
np
.
all
(
np
.
isnan
(
err_vals
)
)
:
raise
ValueError
(
'All err_vals are nans!'
)
return
np
.
nanargmin
(
err_vals
)

Returns the index of the lowest of the passed values . Catches nans etc .


def do_levmarq(s, param_names, damping=0.1, decrease_damp_factor=10.,
        run_length=6, eig_update=True, collect_stats=False, rz_order=0,
        run_type=2, **kwargs):
    """
    Runs Levenberg-Marquardt optimization on a state.

    Convenience wrapper for LMGlobals. Same keyword args, but the defaults
    have been set to useful values for optimizing globals.
    See LMGlobals and LMEngine for documentation.

    See Also
    --------
        do_levmarq_particles : Levenberg-Marquardt optimization of a
            specified set of particles.

        do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
            of all the particles in the state.

        LMGlobals : Optimizer object; the workhorse of do_levmarq.

        LMEngine : Engine superclass for all the optimizers.
    """
    if rz_order > 0:
        aug = AugmentedState(s, param_names, rz_order=rz_order)
        lm = LMAugmentedState(aug, damping=damping, run_length=run_length,
                decrease_damp_factor=decrease_damp_factor, eig_update=
                eig_update, **kwargs)
    else:
        lm = LMGlobals(s, param_names, damping=damping, run_length=run_length,
                decrease_damp_factor=decrease_damp_factor, eig_update=
                eig_update, **kwargs)
    if run_type == 2:
        lm.do_run_2()
    elif run_type == 1:
        lm.do_run_1()
    else:
        raise ValueError('run_type=1,2 only')
    if collect_stats:
        return lm.get_termination_stats()

def
do_levmarq
(
s
,
param_names
,
damping
=
0.1
,
decrease_damp_factor
=
10.
,
run_length
=
6
,
eig_update
=
True
,
collect_stats
=
False
,
rz_order
=
0
,
run_type
=
2
,
*
*
kwargs
)
:
if
rz_order
>
0
:
aug
=
AugmentedState
(
s
,
param_names
,
rz_order
=
rz_order
)
lm
=
LMAugmentedState
(
aug
,
damping
=
damping
,
run_length
=
run_length
,
decrease_damp_factor
=
decrease_damp_factor
,
eig_update
=
eig_update
,
*
*
kwargs
)
else
:
lm
=
LMGlobals
(
s
,
param_names
,
damping
=
damping
,
run_length
=
run_length
,
decrease_damp_factor
=
decrease_damp_factor
,
eig_update
=
eig_update
,
*
*
kwargs
)
if
run_type
==
2
:
lm
.
do_run_2
(
)
elif
run_type
==
1
:
lm
.
do_run_1
(
)
else
:
raise
ValueError
(
'run_type=1,2 only'
)
if
collect_stats
:
return
lm
.
get_termination_stats
(
)

Runs Levenberg - Marquardt optimization on a state .


def do_levmarq_particles(s, particles, damping=1.0, decrease_damp_factor=10.,
        run_length=4, collect_stats=False, max_iter=2, **kwargs):
    """
    Levenberg-Marquardt optimization on a set of particles.

    Convenience wrapper for LMParticles. Same keyword args, but the
    defaults have been set to useful values for optimizing particles.
    See LMParticles and LMEngine for documentation.

    See Also
    --------
        do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
            of all the particles in the state.

        do_levmarq : Levenberg-Marquardt optimization of the entire state;
            useful for optimizing global parameters.

        LMParticles : Optimizer object; the workhorse of do_levmarq_particles.

        LMEngine : Engine superclass for all the optimizers.
    """
    lp = LMParticles(s, particles, damping=damping, run_length=run_length,
            decrease_damp_factor=decrease_damp_factor, max_iter=max_iter,
            **kwargs)
    lp.do_run_2()
    if collect_stats:
        return lp.get_termination_stats()

def do_levmarq_particles ( s , particles , damping = 1.0 , decrease_damp_factor = 10. , run_length = 4 , collect_stats = False , max_iter = 2 , * * kwargs ) : lp = LMParticles ( s , particles , damping = damping , run_length = run_length , decrease_damp_factor = decrease_damp_factor , max_iter = max_iter , * * kwargs ) lp . do_run_2 ( ) if collect_stats : return lp . get_termination_stats ( )

Levenberg - Marquardt optimization on a set of particles .


def do_levmarq_all_particle_groups(s, region_size=40, max_iter=2, damping=1.0,
        decrease_damp_factor=10., run_length=4, collect_stats=False, **kwargs):
    """
    Levenberg-Marquardt optimization for every particle in the state.

    Convenience wrapper for LMParticleGroupCollection. Same keyword args,
    but I've set the defaults to what I've found to be useful values for
    optimizing particles. See LMParticleGroupCollection for documentation.

    See Also
    --------
        do_levmarq_particles : Levenberg-Marquardt optimization of a
            specified set of particles.

        do_levmarq : Levenberg-Marquardt optimization of the entire state;
            useful for optimizing global parameters.

        LMParticleGroupCollection : The workhorse of do_levmarq.

        LMEngine : Engine superclass for all the optimizers.
    """
    lp = LMParticleGroupCollection(s, region_size=region_size, damping=damping,
            run_length=run_length, decrease_damp_factor=decrease_damp_factor,
            get_cos=collect_stats, max_iter=max_iter, **kwargs)
    lp.do_run_2()
    if collect_stats:
        return lp.stats

def do_levmarq_all_particle_groups ( s , region_size = 40 , max_iter = 2 , damping = 1.0 , decrease_damp_factor = 10. , run_length = 4 , collect_stats = False , * * kwargs ) : lp = LMParticleGroupCollection ( s , region_size = region_size , damping = damping , run_length = run_length , decrease_damp_factor = decrease_damp_factor , get_cos = collect_stats , max_iter = max_iter , * * kwargs ) lp . do_run_2 ( ) if collect_stats : return lp . stats

Levenberg - Marquardt optimization for every particle in the state .


def do_levmarq_n_directions(s, directions, max_iter=2, run_length=2,
        damping=1e-3, collect_stats=False, marquardt_damping=True, **kwargs):
    """
    Optimization of a state along a specific set of directions in parameter
    space.

    Parameters
    ----------
        s : :class:`peri.states.State`
            The state to optimize
        directions : np.ndarray
            [n,d] element numpy.ndarray of the n directions in the d-
            dimensional space to optimize along. `directions` is trans-
            formed to a unit vector internally
    Other Parameters
    ----------------
        Any parameters passed to LMEngine.
    """
    # normal = direction / np.sqrt(np.dot(direction, direction))
    normals = np.array([d/np.sqrt(np.dot(d,d)) for d in directions])
    if np.isnan(normals).any():
        raise ValueError('`directions` must not be 0s or contain nan')
    obj = OptState(s, normals)
    lo = LMOptObj(obj, max_iter=max_iter, run_length=run_length, damping=
            damping, marquardt_damping=marquardt_damping, **kwargs)
    lo.do_run_1()
    if collect_stats:
        return lo.get_termination_stats()

Optimization of a state along a specific set of directions in parameter space .


def finish(s, desc='finish', n_loop=4, max_mem=1e9, separate_psf=True,
        fractol=1e-7, errtol=1e-3, dowarn=True):
    """
    Crawls slowly to the minimum-cost state.

    Blocks the global parameters into small enough sections such that each
    can be optimized separately while including all the pixels (i.e. no
    decimation). Optimizes the globals, then the psf separately if desired,
    then particles, then a line minimization along the step direction to
    speed up convergence.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to optimize
        desc : string, optional
            Description to append to the states.save() call every loop.
            Set to `None` to avoid saving. Default is `'finish'`.
        n_loop : Int, optional
            The number of times to loop over in the optimizer. Default is 4.
        max_mem : Numeric, optional
            The maximum amount of memory allowed for the optimizers' J's,
            for both particles & globals. Default is 1e9.
        separate_psf : Bool, optional
            If True, does the psf optimization separately from the rest of
            the globals, since the psf has a more tortuous fit landscape.
            Default is True.
        fractol : Float, optional
            Fractional change in error at which to terminate. Default 1e-4
        errtol : Float, optional
            Absolute change in error at which to terminate. Default 1e-2
        dowarn : Bool, optional
            Whether to log a warning if termination results from finishing
            loops rather than from convergence. Default is True.

    Returns
    -------
        dictionary
            Information about the optimization. Has two keys: ``'converged'``,
            a Bool which of whether optimization stopped due to convergence
            (True) or due to max number of iterations (False), and
            ``'loop_values'``, a [n_loop+1, N] ``numpy.ndarray`` of the
            state's values, at the start of optimization and at the end of
            each loop, before the line minimization.
    """
    values = [np.copy(s.state[s.params])]
    remove_params = s.get('psf').params if separate_psf else None
    # FIXME explicit params
    global_params = name_globals(s, remove_params=remove_params)
    #FIXME this could be done much better, since much of the globals such
    #as the ilm are local. Could be done with sparse matrices and/or taking
    #nearby globals in a group and using the update tile only as the slicer,
    #rather than the full residuals.
    gs = np.floor(max_mem / s.residuals.nbytes).astype('int')
    groups = [global_params[a:a+gs] for a in range(0, len(global_params), gs)]
    CLOG.info('Start  ``finish``:\t{}'.format(s.error))
    for a in range(n_loop):
        start_err = s.error
        #1. Min globals:
        for g in groups:
            do_levmarq(s, g, damping=0.1, decrease_damp_factor=20.,
                    max_iter=1, max_mem=max_mem, eig_update=False)
        if separate_psf:
            do_levmarq(s, remove_params, max_mem=max_mem, max_iter=4,
                    eig_update=False)
        CLOG.info('Globals,   loop {}:\t{}'.format(a, s.error))
        if desc is not None:
            states.save(s, desc=desc)
        #2. Min particles
        do_levmarq_all_particle_groups(s, max_iter=1, max_mem=max_mem)
        CLOG.info('Particles, loop {}:\t{}'.format(a, s.error))
        if desc is not None:
            states.save(s, desc=desc)
        #3. Append vals, line min:
        values.append(np.copy(s.state[s.params]))
        # dv = (np.array(values[1:]) - np.array(values[0]))[-3:]
        # do_levmarq_n_directions(s, dv, damping=1e-2, max_iter=2, errtol=3e-4)
        # CLOG.info('Line min., loop {}:\t{}'.format(a, s.error))
        # if desc is not None:
            # states.save(s, desc=desc)
        #4. terminate?
        new_err = s.error
        derr = start_err - new_err
        dobreak = (derr/new_err < fractol) or (derr < errtol)
        if dobreak:
            break

    if dowarn and (not dobreak):
        CLOG.warn('finish() did not converge; consider re-running')
    return {'converged':dobreak, 'loop_values':np.array(values)}

Crawls slowly to the minimum - cost state .


def fit_comp(new_comp, old_comp, **kwargs):
    """
    Fits a new component to an old component

    Calls do_levmarq to match the .get() fields of the two objects. The
    parameters of new_comp are modified in place.

    Parameters
    ----------
    new_comp : :class:`peri.comps.comp`
        The new object, whose parameters to update to fit the field of
        `old_comp`. Must have a .get() attribute which returns an ndarray
    old_comp : peri.comp
        The old ilm to match to.

    Other Parameters
    ----------------
        Any keyword arguments to be passed to the optimizer LMGlobals
        through do_levmarq.

    See Also
    --------
    do_levmarq : Levenberg-Marquardt minimization using a random subset
        of the image pixels.
    """
    #resetting the category to ilm:
    new_cat = new_comp.category
    new_comp.category = 'ilm'
    fake_s = states.ImageState(Image(old_comp.get().copy()), [new_comp], pad=0,
            mdl=mdl.SmoothFieldModel())
    do_levmarq(fake_s, new_comp.params, **kwargs)
    new_comp.category = new_cat

def
fit_comp
(
new_comp
,
old_comp
,
*
*
kwargs
)
:
#resetting the category to ilm:
new_cat
=
new_comp
.
category
new_comp
.
category
=
'ilm'
fake_s
=
states
.
ImageState
(
Image
(
old_comp
.
get
(
)
.
copy
(
)
)
,
[
new_comp
]
,
pad
=
0
,
mdl
=
mdl
.
SmoothFieldModel
(
)
)
do_levmarq
(
fake_s
,
new_comp
.
params
,
*
*
kwargs
)
new_comp
.
category
=
new_cat


def reset(self, new_damping=None):
        """
        Keeps all user supplied options the same, but resets counters etc.
        """
        self._num_iter = 0
        self._inner_run_counter = 0
        self._J_update_counter = self.update_J_frequency
        self._fresh_JTJ = False
        self._has_run = False
        if new_damping is not None:
            self.damping = np.array(new_damping).astype('float')
        self._set_err_paramvals()

def reset ( self , new_damping = None ) : self . _num_iter = 0 self . _inner_run_counter = 0 self . _J_update_counter = self . update_J_frequency self . _fresh_JTJ = False self . _has_run = False if new_damping is not None : self . damping = np . array ( new_damping ) . astype ( 'float' ) self . _set_err_paramvals ( )

Keeps all user supplied options the same but resets counters etc .


def do_run_1(self):
        """
        LM run, evaluating 1 step at a time.

        Broyden or eigendirection updates replace full-J updates until
        a full-J update occurs. Does not run with the calculated J (no
        internal run).
        """
        while not self.check_terminate():
            self._has_run = True
            self._run1()
            self._num_iter += 1; self._inner_run_counter += 1

def do_run_1 ( self ) : while not self . check_terminate ( ) : self . _has_run = True self . _run1 ( ) self . _num_iter += 1 self . _inner_run_counter += 1


def _run1(self):
        """workhorse for do_run_1"""
        if self.check_update_J():
            self.update_J()
        else:
            if self.check_Broyden_J():
                self.update_Broyden_J()
            if self.check_update_eig_J():
                self.update_eig_J()

        #1. Assuming that J starts updated:
        delta_vals = self.find_LM_updates(self.calc_grad())

        #2. Increase damping until we get a good step:
        er1 = self.update_function(self.param_vals + delta_vals)
        good_step = (find_best_step([self.error, er1]) == 1)
        if not good_step:
            er0 = self.update_function(self.param_vals)
            if np.abs(er0 -self.error)/er0 > 1e-7:
                raise RuntimeError('Function updates are not exact.')
            CLOG.debug('Bad step, increasing damping')
            CLOG.debug('\t\t%f\t%f' % (self.error, er1))
            grad = self.calc_grad()
            for _try in range(self._max_inner_loop):
                self.increase_damping()
                delta_vals = self.find_LM_updates(grad)
                er1 = self.update_function(self.param_vals + delta_vals)
                good_step = (find_best_step([self.error, er1]) == 1)
                if good_step:
                    break
            else:
                er0 = self.update_function(self.param_vals)
                CLOG.warn('Stuck!')
                if np.abs(er0 -self.error)/er0 > 1e-7:
                    raise RuntimeError('Function updates are not exact.')

        #state is updated, now params:
        if good_step:
            self._last_error = self.error
            self.error = er1
            CLOG.debug('Good step\t%f\t%f' % (self._last_error, self.error))
            self.update_param_vals(delta_vals, incremental=True)
            self.decrease_damping()


def _run2(self):
        """Workhorse for do_run_2"""
        if self.check_update_J():
            self.update_J()
        else:
            if self.check_Broyden_J():
                self.update_Broyden_J()
            if self.check_update_eig_J():
                self.update_eig_J()

        #0. Find _last_residuals, _last_error, etc:
        _last_residuals = self.calc_residuals().copy()
        _last_error = 1*self.error
        _last_vals = self.param_vals.copy()

        #1. Calculate 2 possible steps
        delta_params_1 = self.find_LM_updates(self.calc_grad(),
                do_correct_damping=False)
        self.decrease_damping()
        delta_params_2 = self.find_LM_updates(self.calc_grad(),
                do_correct_damping=False)
        self.decrease_damping(undo_decrease=True)

        #2. Check which step is best:
        er1 = self.update_function(self.param_vals + delta_params_1)
        er2 = self.update_function(self.param_vals + delta_params_2)

        triplet = (self.error, er1, er2)
        best_step = find_best_step(triplet)
        if best_step == 0:
            #Both bad steps, put back & increase damping:
            _ = self.update_function(self.param_vals.copy())
            grad = self.calc_grad()
            CLOG.debug('Bad step, increasing damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            for _try in range(self._max_inner_loop):
                self.increase_damping()
                delta_vals = self.find_LM_updates(grad)
                er_new = self.update_function(self.param_vals + delta_vals)
                good_step = er_new < self.error
                if good_step:
                    #Update params, error, break:
                    self.update_param_vals(delta_vals, incremental=True)
                    self.error = er_new
                    CLOG.debug('Sufficiently increased damping')
                    CLOG.debug('%f\t%f' % (triplet[0], self.error))
                    break
            else: #for-break-else
                #Throw a warning, put back the parameters
                CLOG.warn('Stuck!')
                self.error = self.update_function(self.param_vals.copy())

        elif best_step == 1:
            #er1 <= er2:
            good_step = True
            CLOG.debug('Good step, same damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            #Update to er1 params:
            er1_1 = self.update_function(self.param_vals + delta_params_1)
            if np.abs(er1_1 - er1) > 1e-6:
                raise RuntimeError('Function updates are not exact.')
            self.update_param_vals(delta_params_1, incremental=True)
            self.error = er1

        elif best_step == 2:
            #er2 < er1:
            good_step = True
            self.error = er2
            CLOG.debug('Good step, decreasing damping')
            CLOG.debug('%f\t%f\t%f' % triplet)
            #-we're already at the correct parameters
            self.update_param_vals(delta_params_2, incremental=True)
            self.decrease_damping()

        #3. Run with current J, damping; update what we need to::
        if good_step:
            self._last_residuals = _last_residuals
            self._last_error = _last_error
            self._last_vals = _last_vals
            self.error
            self.do_internal_run(initial_count=1)


def do_internal_run(self, initial_count=0, subblock=None, update_derr=True):
        """
        Takes more steps without calculating J again.

        Given a fixed damping, J, JTJ, iterates calculating steps, with
        optional Broyden or eigendirection updates. Iterates either until
        a bad step is taken or for self.run_length times.
        Called internally by do_run_2() but is also useful on its own.

        Parameters
        ----------
            initial_count : Int, optional
                The initial count of the run. Default is 0. Increasing from
                0 effectively temporarily decreases run_length.
            subblock : None or np.ndarray of bools, optional
                If not None, a boolean mask which determines which sub-
                block of parameters to run over. Default is None, i.e.
                all the parameters.
            update_derr : Bool, optional
                Set to False to not update the variable that determines
                delta_err, preventing premature termination through errtol.

        Notes
        -----
        It might be good to do something similar to update_derr with the
        parameter values, but this is trickier because of Broyden updates
        and _fresh_J.
        """
        self._inner_run_counter = initial_count; good_step = True
        n_good_steps = 0
        CLOG.debug('Running...')

        _last_residuals = self.calc_residuals().copy()
        while ((self._inner_run_counter < self.run_length) & good_step &
                (not self.check_terminate())):
            #1. Checking if we update J
            if self.check_Broyden_J() and self._inner_run_counter != 0:
                self.update_Broyden_J()
            if self.check_update_eig_J() and self._inner_run_counter != 0:
                self.update_eig_J()

            #2. Getting parameters, error
            er0 = 1*self.error
            delta_vals = self.find_LM_updates(self.calc_grad(),
                    do_correct_damping=False, subblock=subblock)
            er1 = self.update_function(self.param_vals + delta_vals)
            good_step = er1 < er0

            if good_step:
                n_good_steps += 1
                CLOG.debug('%f\t%f' % (er0, er1))
                #Updating:
                self.update_param_vals(delta_vals, incremental=True)
                self._last_residuals = _last_residuals.copy()
                if update_derr:
                    self._last_error = er0
                self.error = er1

                _last_residuals = self.calc_residuals().copy()
            else:
                er0_0 = self.update_function(self.param_vals)
                CLOG.debug('Bad step!')
                if np.abs(er0 - er0_0) > 1e-6:
                    raise RuntimeError('Function updates are not exact.')

            self._inner_run_counter += 1
        return n_good_steps

def
do_internal_run
(
self
,
initial_count
=
0
,
subblock
=
,
update_derr
=
True
)
:
self
.
_inner_run_counter
=
initial_count
good_step
=
True
n_good_steps
=
0
CLOG
.
debug
(
'Running...'
)
_last_residuals
=
self
.
calc_residuals
(
)
.
copy
(
)
while
(
(
self
.
_inner_run_counter
<
self
.
run_length
)
&
good_step
&
(
not
self
.
check_terminate
(
)
)
)
:
#1. Checking if we update J
if
self
.
check_Broyden_J
(
)
and
self
.
_inner_run_counter
!=
0
:
self
.
update_Broyden_J
(
)
if
self
.
check_update_eig_J
(
)
and
self
.
_inner_run_counter
!=
0
:
self
.
update_eig_J
(
)
#2. Getting parameters, error
er0
=
1
*
self
.
error
delta_vals
=
self
.
find_LM_updates
(
self
.
calc_grad
(
)
,
do_correct_damping
=
False
,
subblock
=
subblock
)
er1
=
self
.
update_function
(
self
.
param_vals
+
delta_vals
)
good_step
=
er1
<
er0
if
good_step
:
n_good_steps
+=
1
CLOG
.
debug
(
'%f\t%f'
%
(
er0
,
er1
)
)
#Updating:
self
.
update_param_vals
(
delta_vals
,
incremental
=
True
)
self
.
_last_residuals
=
_last_residuals
.
copy
(
)
if
update_derr
:
self
.
_last_error
=
er0
self
.
error
=
er1
_last_residuals
=
self
.
calc_residuals
(
)
.
copy
(
)
else
:
er0_0
=
self
.
update_function
(
self
.
param_vals
)
CLOG
.
debug
(
'Bad step!'
)
if
np
.
abs
(
er0
er0_0
)
>
1e-6
:
raise
RuntimeError
(
'Function updates are not exact.'
)
self
.
_inner_run_counter
+=
1
return
n_good_steps

Takes more steps without calculating J again .


def find_LM_updates(self, grad, do_correct_damping=True, subblock=None):
        """
        Calculates LM updates, with or without the acceleration correction.

        Parameters
        ----------
            grad : numpy.ndarray
                The gradient of the model cost.
            do_correct_damping : Bool, optional
                If `self.use_accel`, then set to True to correct damping
                if the acceleration correction is too big. Default is True
                Does nothing is `self.use_accel` is False
            subblock : slice, numpy.ndarray, or None, optional
                Set to a slice or a valide numpy.ndarray to use only a
                certain subset of the parameters. Default is None, i.e.
                use all the parameters.

        Returns
        -------
            delta : numpy.ndarray
                The Levenberg-Marquadt step, relative to the old
                parameters. Size is always self.param_vals.size.
        """
        if subblock is not None:
            if (subblock.sum() == 0) or (subblock.size == 0):
                CLOG.fatal('Empty subblock in find_LM_updates')
                raise ValueError('Empty sub-block')
            j = self.J[subblock]
            JTJ = np.dot(j, j.T)
            damped_JTJ = self._calc_damped_jtj(JTJ, subblock=subblock)
            grad = grad[subblock]  #select the subblock of the grad
        else:
            damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)

        delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if self.use_accel:
            accel_correction = self.calc_accel_correction(damped_JTJ, delta)
            nrm_d0 = np.sqrt(np.sum(delta**2))
            nrm_corr = np.sqrt(np.sum(accel_correction**2))
            CLOG.debug('|correction| / |LM step|\t%e' % (nrm_corr/nrm_d0))
            if nrm_corr/nrm_d0 < self.max_accel_correction:
                delta += accel_correction
            elif do_correct_damping:
                CLOG.debug('Untrustworthy step! Increasing damping...')
                self.increase_damping()
                damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)
                delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if np.any(np.isnan(delta)):
            CLOG.fatal('Calculated steps have nans!?')
            raise FloatingPointError('Calculated steps have nans!?')
        return delta

Calculates LM updates with or without the acceleration correction .


def update_param_vals(self, new_vals, incremental=False):
        """
        Updates the current set of parameter values and previous values,
        sets a flag to re-calculate J.

        Parameters
        ----------
            new_vals : numpy.ndarray
                The new values to update to
            incremental : Bool, optional
                Set to True to make it an incremental update relative
                to the old parameters. Default is False
        """
        self._last_vals = self.param_vals.copy()
        if incremental:
            self.param_vals += new_vals
        else:
            self.param_vals = new_vals.copy()
        #And we've updated, so JTJ is no longer valid:
        self._fresh_JTJ = False

Updates the current set of parameter values and previous values sets a flag to re - calculate J .


def get_termination_stats(self, get_cos=True):
        """
        Returns a dict of termination statistics

        Parameters
        ----------
            get_cos : Bool, optional
                Whether or not to calcualte the cosine of the residuals
                with the tangent plane of the model using the current J.
                The calculation may take some time. Default is True

        Returns
        -------
            dict
                Has keys
                    delta_vals  : The last change in parameter values.
                    delta_err   : The last change in the error.
                    exp_err     : The expected (last) change in the error.
                    frac_err    : The fractional change in the error.
                    num_iter    : The number of iterations completed.
                    error       : The current error.
        """
        delta_vals = self._last_vals - self.param_vals
        delta_err = self._last_error - self.error
        frac_err = delta_err / self.error
        to_return = {'delta_vals':delta_vals, 'delta_err':delta_err,
                'num_iter':1*self._num_iter, 'frac_err':frac_err,
                'error':self.error, 'exp_err':self._exp_err}
        if get_cos:
            model_cosine = self.calc_model_cosine()
            to_return.update({'model_cosine':model_cosine})
        return to_return

def get_termination_stats ( self , get_cos = True ) : delta_vals = self . _last_vals - self . param_vals delta_err = self . _last_error - self . error frac_err = delta_err / self . error to_return = { 'delta_vals' : delta_vals , 'delta_err' : delta_err , 'num_iter' : 1 * self . _num_iter , 'frac_err' : frac_err , 'error' : self . error , 'exp_err' : self . _exp_err } if get_cos : model_cosine = self . calc_model_cosine ( ) to_return . update ( { 'model_cosine' : model_cosine } ) return to_return


def check_completion(self):
        """
        Returns a Bool of whether the algorithm has found a satisfactory minimum
        """
        terminate = False
        term_dict = self.get_termination_stats(get_cos=self.costol is not None)
        terminate |= np.all(np.abs(term_dict['delta_vals']) < self.paramtol)
        terminate |= (term_dict['delta_err'] < self.errtol)
        terminate |= (term_dict['exp_err'] < self.exptol)
        terminate |= (term_dict['frac_err'] < self.fractol)
        if self.costol is not None:
            terminate |= (curcos < term_dict['model_cosine'])

        return terminate

def check_completion ( self ) : terminate = False term_dict = self . get_termination_stats ( get_cos = self . costol is not None ) terminate |= np . all ( np . abs ( term_dict [ 'delta_vals' ] ) < self . paramtol ) terminate |= ( term_dict [ 'delta_err' ] < self . errtol ) terminate |= ( term_dict [ 'exp_err' ] < self . exptol ) terminate |= ( term_dict [ 'frac_err' ] < self . fractol ) if self . costol is not None : terminate |= ( curcos < term_dict [ 'model_cosine' ] ) return terminate


def check_terminate(self):
        """
        Returns a Bool of whether to terminate.

        Checks whether a satisfactory minimum has been found or whether
        too many iterations have occurred.
        """
        if not self._has_run:
            return False
        else:
            #1-3. errtol, paramtol, model cosine low enough?
            terminate = self.check_completion()

            #4. too many iterations??
            terminate |= (self._num_iter >= self.max_iter)
            return terminate

Returns a Bool of whether to terminate .


def check_update_J(self):
        """
        Checks if the full J should be updated.

        Right now, just updates after update_J_frequency loops
        """
        self._J_update_counter += 1
        update = self._J_update_counter >= self.update_J_frequency
        return update & (not self._fresh_JTJ)

def check_update_J ( self ) : self . _J_update_counter += 1 update = self . _J_update_counter >= self . update_J_frequency return update & ( not self . _fresh_JTJ )

Checks if the full J should be updated .


def update_J(self):
        """Updates J, JTJ, and internal counters."""
        self.calc_J()
        # np.dot(j, j.T) is slightly faster but 2x as much mem
        step = np.ceil(1e-2 * self.J.shape[1]).astype('int')  # 1% more mem...
        self.JTJ = low_mem_sq(self.J, step=step)
        #copies still, since J is not C -ordered but a slice of j_e...
        #doing self.J.copy() works but takes 2x as much ram..
        self._fresh_JTJ = True
        self._J_update_counter = 0
        if np.any(np.isnan(self.JTJ)):
            raise FloatingPointError('J, JTJ have nans.')
        #Update self._exp_err
        self._exp_err = self.error - self.find_expected_error(delta_params='perfect')


def update_Broyden_J(self):
        """Execute a Broyden update of J"""
        CLOG.debug('Broyden update.')
        delta_vals = self.param_vals - self._last_vals
        delta_residuals = self.calc_residuals() - self._last_residuals
        nrm = np.sqrt(np.dot(delta_vals, delta_vals))
        direction = delta_vals / nrm
        vals = delta_residuals / nrm
        self._rank_1_J_update(direction, vals)
        self.JTJ = np.dot(self.J, self.J.T)

def
update_Broyden_J
(
self
)
:
CLOG
.
debug
(
'Broyden update.'
)
delta_vals
=
self
.
param_vals
self
.
_last_vals
delta_residuals
=
self
.
calc_residuals
(
)
self
.
_last_residuals
nrm
=
np
.
sqrt
(
np
.
dot
(
delta_vals
,
delta_vals
)
)
direction
=
delta_vals
/
nrm
vals
=
delta_residuals
/
nrm
self
.
_rank_1_J_update
(
direction
,
vals
)
self
.
JTJ
=
np
.
dot
(
self
.
J
,
self
.
J
.
T
)


def update_eig_J(self):
        """Execute an eigen update of J"""
        CLOG.debug('Eigen update.')
        vls, vcs = np.linalg.eigh(self.JTJ)
        res0 = self.calc_residuals()
        for a in range(min([self.num_eig_dirs, vls.size])):
            #1. Finding stiff directions
            stif_dir = vcs[-(a+1)] #already normalized

            #2. Evaluating derivative along that direction, we'll use dl=5e-4:
            dl = self.eig_dl #1e-5
            _ = self.update_function(self.param_vals + dl*stif_dir)
            res1 = self.calc_residuals()

            #3. Updating
            grad_stif = (res1-res0)/dl
            self._rank_1_J_update(stif_dir, grad_stif)

        self.JTJ = np.dot(self.J, self.J.T)
        #Putting the parameters back:
        _ = self.update_function(self.param_vals)


def calc_accel_correction(self, damped_JTJ, delta0):
        """
        Geodesic acceleration correction to the LM step.

        Parameters
        ----------
            damped_JTJ : numpy.ndarray
                The damped JTJ used to calculate the initial step.
            delta0 : numpy.ndarray
                The initial LM step.

        Returns
        -------
            corr : numpy.ndarray
                The correction to the original LM step.
        """
        #Get the derivative:
        _ = self.update_function(self.param_vals)
        rm0 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals + delta0)
        rm1 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals - delta0)
        rm2 = self.calc_residuals().copy()
        der2 = (rm2 + rm1 - 2*rm0)

        corr, res, rank, s = np.linalg.lstsq(damped_JTJ, np.dot(self.J, der2),
                rcond=self.min_eigval)
        corr *= -0.5
        return corr

def
calc_accel_correction
(
self
,
damped_JTJ
,
delta0
)
:
#Get the derivative:
_
=
self
.
update_function
(
self
.
param_vals
)
rm0
=
self
.
calc_residuals
(
)
.
copy
(
)
_
=
self
.
update_function
(
self
.
param_vals
+
delta0
)
rm1
=
self
.
calc_residuals
(
)
.
copy
(
)
_
=
self
.
update_function
(
self
.
param_vals
delta0
)
rm2
=
self
.
calc_residuals
(
)
.
copy
(
)
der2
=
(
rm2
+
rm1
2
*
rm0
)
corr
,
res
,
rank
,
s
=
np
.
linalg
.
lstsq
(
damped_JTJ
,
np
.
dot
(
self
.
J
,
der2
)
,
rcond
=
self
.
min_eigval
)
corr
*=
0.5
return
corr

Geodesic acceleration correction to the LM step .


def calc_J(self):
        """Updates self.J, returns nothing"""
        del self.J
        self.J = np.zeros([self.param_vals.size, self.data.size])
        dp = np.zeros_like(self.param_vals)
        f0 = self.model.copy()
        for a in range(self.param_vals.size):
            dp *= 0
            dp[a] = self.dl[a]
            f1 = self.func(self.param_vals + dp, *self.func_args, **self.func_kwargs)
            grad_func = (f1 - f0) / dp[a]
            #J = grad(residuals) = -grad(model)
            self.J[a] = -grad_func

def
calc_J
(
self
)
:
del
self
.
J
self
.
J
=
np
.
zeros
(
[
self
.
param_vals
.
size
,
self
.
data
.
size
]
)
dp
=
np
.
zeros_like
(
self
.
param_vals
)
f0
=
self
.
model
.
copy
(
)
for
a
in
range
(
self
.
param_vals
.
size
)
:
dp
*=
0
dp
[
a
]
=
self
.
dl
[
a
]
f1
=
self
.
func
(
self
.
param_vals
+
dp
,
*
self
.
func_args
,
*
*
self
.
func_kwargs
)
grad_func
=
(
f1
f0
)
/
dp
[
a
]
#J = grad(residuals) = -grad(model)
self
.
J
[
a
]
=
grad_func


def update_function(self, param_vals):
        """Takes an array param_vals, updates function, returns the new error"""
        self.model = self.func(param_vals, *self.func_args, **self.func_kwargs)
        d = self.calc_residuals()
        return np.dot(d.flat, d.flat)

def update_function ( self , param_vals ) : self . model = self . func ( param_vals , * self . func_args , * * self . func_kwargs ) d = self . calc_residuals ( ) return np . dot ( d . flat , d . flat )

Takes an array param_vals updates function returns the new error


def update_function(self, param_vals):
        """Updates the opt_obj, returns new error."""
        self.opt_obj.update_function(param_vals)
        return self.opt_obj.get_error()

def update_function ( self , param_vals ) : self . opt_obj . update_function ( param_vals ) return self . opt_obj . get_error ( )


def calc_J(self):
        """Calculates J along the direction."""
        r0 = self.state.residuals.copy().ravel()
        dl = np.zeros(self.param_vals.size)
        p0 = self.param_vals.copy()
        J = []
        for a in range(self.param_vals.size):
            dl *= 0
            dl[a] += self.dl
            self.update_function(p0 + dl)
            r1 = self.state.residuals.copy().ravel()
            J.append( (r1-r0)/self.dl)
        self.update_function(p0)
        return np.array(J)

def calc_J ( self ) : r0 = self . state . residuals . copy ( ) . ravel ( ) dl = np . zeros ( self . param_vals . size ) p0 = self . param_vals . copy ( ) J = [ ] for a in range ( self . param_vals . size ) : dl *= 0 dl [ a ] += self . dl self . update_function ( p0 + dl ) r1 = self . state . residuals . copy ( ) . ravel ( ) J . append ( ( r1 - r0 ) / self . dl ) self . update_function ( p0 ) return np . array ( J )


def reset(self, new_region_size=None, do_calc_size=True, new_damping=None,
            new_max_mem=None):
        """
        Resets the particle groups and optionally the region size and damping.

        Parameters
        ----------
            new_region_size : : Int or 3-element list-like of ints, optional
                The region size for sub-blocking particles. Default is 40
            do_calc_size : Bool, optional
                If True, calculates the region size internally based on
                the maximum allowed memory. Default is True
            new_damping : Float or None, optional
                The new damping of the optimizer. Set to None to leave
                as the default for LMParticles. Default is None.
            new_max_mem : Numeric, optional
                The maximum allowed memory for J to occupy. Default is 1e9
        """
        if new_region_size is not None:
            self.region_size = new_region_size
        if new_max_mem != None:
            self.max_mem = new_max_mem
        if do_calc_size:
            self.region_size = calc_particle_group_region_size(self.state,
                    region_size=self.region_size, max_mem=self.max_mem)
        self.stats = []
        self.particle_groups = separate_particles_into_groups(self.state,
                self.region_size, doshift='rand')
        if new_damping is not None:
            self._kwargs.update({'damping':new_damping})
        if self.save_J:
            if len(self.particle_groups) > 90:
                CLOG.warn('Attempting to create many open files. Consider increasing max_mem and/or region_size to avoid crashes.')
            self._tempfiles = []
            self._has_saved_J = []
            for a in range(len(self.particle_groups)):
                #TemporaryFile is automatically deleted
                for _ in ['j','tile']:
                    self._tempfiles.append(tempfile.TemporaryFile(dir=os.getcwd()))
                self._has_saved_J.append(False)

Resets the particle groups and optionally the region size and damping .


def _do_run(self, mode='1'):
        """workhorse for the self.do_run_xx methods."""
        for a in range(len(self.particle_groups)):
            group = self.particle_groups[a]
            lp = LMParticles(self.state, group, **self._kwargs)
            if mode == 'internal':
                lp.J, lp.JTJ, lp._dif_tile = self._load_j_diftile(a)

            if mode == '1':
                lp.do_run_1()
            if mode == '2':
                lp.do_run_2()
            if mode == 'internal':
                lp.do_internal_run()

            self.stats.append(lp.get_termination_stats(get_cos=self.get_cos))
            if self.save_J and (mode != 'internal'):
                self._dump_j_diftile(a, lp.J, lp._dif_tile)
                self._has_saved_J[a] = True

def _do_run ( self , mode = '1' ) : for a in range ( len ( self . particle_groups ) ) : group = self . particle_groups [ a ] lp = LMParticles ( self . state , group , * * self . _kwargs ) if mode == 'internal' : lp . J , lp . JTJ , lp . _dif_tile = self . _load_j_diftile ( a ) if mode == '1' : lp . do_run_1 ( ) if mode == '2' : lp . do_run_2 ( ) if mode == 'internal' : lp . do_internal_run ( ) self . stats . append ( lp . get_termination_stats ( get_cos = self . get_cos ) ) if self . save_J and ( mode != 'internal' ) : self . _dump_j_diftile ( a , lp . J , lp . _dif_tile ) self . _has_saved_J [ a ] = True

workhorse for the self . do_run_xx methods .


def do_internal_run(self):
        """Calls LMParticles.do_internal_run for each group of particles."""
        if not self.save_J:
            raise RuntimeError('self.save_J=True required for do_internal_run()')
        if not np.all(self._has_saved_J):
            raise RuntimeError('J, JTJ have not been pre-computed. Call do_run_1 or do_run_2')
        self._do_run(mode='internal')

Calls LMParticles . do_internal_run for each group of particles .


def reset(self):
        """
        Resets the initial radii used for updating the particles. Call
        if any of the particle radii or positions have been changed
        external to the augmented state.
        """
        inds = list(range(self.state.obj_get_positions().shape[0]))
        self._rad_nms = self.state.param_particle_rad(inds)
        self._pos_nms = self.state.param_particle_pos(inds)
        self._initial_rad = np.copy(self.state.state[self._rad_nms])
        self._initial_pos = np.copy(self.state.state[self._pos_nms]).reshape((-1,3))
        self.param_vals[self.rscale_mask] = 0

def reset ( self ) : inds = list ( range ( self . state . obj_get_positions ( ) . shape [ 0 ] ) ) self . _rad_nms = self . state . param_particle_rad ( inds ) self . _pos_nms = self . state . param_particle_pos ( inds ) self . _initial_rad = np . copy ( self . state . state [ self . _rad_nms ] ) self . _initial_pos = np . copy ( self . state . state [ self . _pos_nms ] ) . reshape ( ( - 1 , 3 ) ) self . param_vals [ self . rscale_mask ] = 0

Resets the initial radii used for updating the particles . Call if any of the particle radii or positions have been changed external to the augmented state .


def reset(self, **kwargs):
        """Resets the aug_state and the LMEngine"""
        self.aug_state.reset()
        super(LMAugmentedState, self).reset(**kwargs)

def reset ( self , * * kwargs ) : self . aug_state . reset ( ) super ( LMAugmentedState , self ) . reset ( * * kwargs )


def get_shares(self):
    '''
      Returns an object with a the numbers of shares a link has had using
      Buffer.

      www will be stripped, but other subdomains will not.
    '''

    self.shares = self.api.get(url=PATHS['GET_SHARES'] % self.url)['shares']

    return self.shares

def get_shares ( self ) : self . shares = self . api . get ( url = PATHS [ 'GET_SHARES' ] % self . url ) [ 'shares' ] return self . shares

Returns an object with a the numbers of shares a link has had using Buffer .


def sample(field, inds=None, slicer=None, flat=True):
    """
    Take a sample from a field given flat indices or a shaped slice

    Parameters
    -----------
    inds : list of indices
        One dimensional (raveled) indices to return from the field

    slicer : slice object
        A shaped (3D) slicer that returns a section of image

    flat : boolean
        Whether to flatten the sampled item before returning
    """
    if inds is not None:
        out = field.ravel()[inds]
    elif slicer is not None:
        out = field[slicer].ravel()
    else:
        out = field

    if flat:
        return out.ravel()
    return out

def sample ( field , inds = None , slicer = None , flat = True ) : if inds is not None : out = field . ravel ( ) [ inds ] elif slicer is not None : out = field [ slicer ] . ravel ( ) else : out = field if flat : return out . ravel ( ) return out


def update(self, params, values):
        """
        Update a single parameter or group of parameters ``params``
        with ``values``.

        Parameters
        ----------
        params : string or list of strings
            Parameter names which to update

        value : number or list of numbers
            Values of those parameters which to update
        """
        return super(State, self).update(params, values)

def update ( self , params , values ) : return super ( State , self ) . update ( params , values )

Update a single parameter or group of parameters params with values .


def build_funcs(self):
        """
        Here, we build gradient and hessian functions based on the properties
        of a state that are generally wanted. For each one, we fill in _grad or
        _hess with a function that takes care of various options such as
        slicing and flattening. For example, `m` below takes the model, selects
        different indices from it, maybe flattens it and copies it. This is
        then used in the fisherinformation, gradmodel, and hessmodel functions.
        """
        # create essentially lambda functions, but with a nice signature
        def m(inds=None, slicer=None, flat=True):
            return sample(self.model, inds=inds, slicer=slicer, flat=flat).copy()

        def r(inds=None, slicer=None, flat=True):
            return sample(self.residuals, inds=inds, slicer=slicer, flat=flat).copy()

        def l():
            return self.loglikelihood

        def r_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return r(**kwargs), np.copy(self.error)

        def m_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return m(**kwargs), np.copy(self.error)

        # set the member functions using partial
        self.fisherinformation = partial(self._jtj, funct=m)
        self.gradloglikelihood = partial(self._grad, funct=l)
        self.hessloglikelihood = partial(self._hess, funct=l)
        self.gradmodel = partial(self._grad, funct=m)
        self.hessmodel = partial(self._hess, funct=m)
        self.JTJ = partial(self._jtj, funct=r)
        self.J = partial(self._grad, funct=r)
        self.J_e = partial(self._grad, funct=r_e, nout=2)
        self.gradmodel_e = partial(self._grad, funct=m_e, nout=2)

        # add the appropriate documentation to the following functions
        self.fisherinformation.__doc__ = _graddoc + _sampledoc
        self.gradloglikelihood.__doc__ = _graddoc
        self.hessloglikelihood.__doc__ = _graddoc
        self.gradmodel.__doc__ = _graddoc + _sampledoc
        self.hessmodel.__doc__ = _graddoc + _sampledoc
        self.JTJ.__doc__ = _graddoc + _sampledoc
        self.J.__doc__ = _graddoc + _sampledoc

        # add documentation to the private functions as well. this is done
        # slightly differently, hence the function call
        self._dograddoc(self._grad_one_param)
        self._dograddoc(self._hess_two_param)
        self._dograddoc(self._grad)
        self._dograddoc(self._hess)

        # the state object is a workaround so that other interfaces still
        # work. this should probably be removed in the long run
        class _Statewrap(object):
            def __init__(self, obj):
                self.obj = obj
            def __getitem__(self, d=None):
                if d is None:
                    d = self.obj.params
                return util.delistify(self.obj.get_values(d), d)

        self.state = _Statewrap(self)

Here we build gradient and hessian functions based on the properties of a state that are generally wanted . For each one we fill in _grad or _hess with a function that takes care of various options such as slicing and flattening . For example m below takes the model selects different indices from it maybe flattens it and copies it . This is then used in the fisherinformation gradmodel and hessmodel functions .


def set_model(self, mdl):
        """
        Setup the image model formation equation and corresponding objects into
        their various objects. `mdl` is a `peri.models.Model` object
        """
        self.mdl = mdl
        self.mdl.check_inputs(self.comps)

        for c in self.comps:
            setattr(self, '_comp_'+c.category, c)

def set_model ( self , mdl ) : self . mdl = mdl self . mdl . check_inputs ( self . comps ) for c in self . comps : setattr ( self , '_comp_' + c . category , c )

Setup the image model formation equation and corresponding objects into their various objects . mdl is a peri . models . Model object


def model_to_data(self, sigma=0.0):
        """ Switch out the data for the model's recreation of the data. """
        im = self.model.copy()
        im += sigma*np.random.randn(*im.shape)
        self.set_image(util.NullImage(image=im))

def model_to_data ( self , sigma = 0.0 ) : im = self . model . copy ( ) im += sigma * np . random . randn ( * im . shape ) self . set_image ( util . NullImage ( image = im ) )

Switch out the data for the model s recreation of the data .


def get_update_io_tiles(self, params, values):
        """
        Get the tiles corresponding to a particular section of image needed to
        be updated. Inputs are the parameters and values. Returned is the
        padded tile, inner tile, and slicer to go between, but accounting for
        wrap with the edge of the image as necessary.
        """
        # get the affected area of the model image
        otile = self.get_update_tile(params, values)
        if otile is None:
            return [None]*3
        ptile = self.get_padding_size(otile) or util.Tile(0, dim=otile.dim)

        otile = util.Tile.intersection(otile, self.oshape)

        if (otile.shape <= 0).any():
            raise UpdateError("update triggered invalid tile size")

        if (ptile.shape < 0).any() or (ptile.shape > self.oshape.shape).any():
            raise UpdateError("update triggered invalid padding tile size")

        # now remove the part of the tile that is outside the image and pad the
        # interior part with that overhang. reflect the necessary padding back
        # into the image itself for the outer slice which we will call outer
        outer = otile.pad((ptile.shape+1)//2)
        inner, outer = outer.reflect_overhang(self.oshape)
        iotile = inner.translate(-outer.l)

        outer = util.Tile.intersection(outer, self.oshape)
        inner = util.Tile.intersection(inner, self.oshape)
        return outer, inner, iotile

Get the tiles corresponding to a particular section of image needed to be updated . Inputs are the parameters and values . Returned is the padded tile inner tile and slicer to go between but accounting for wrap with the edge of the image as necessary .


def get(self, name):
        """ Return component by category name """
        for c in self.comps:
            if c.category == name:
                return c
        return None

def get ( self , name ) : for c in self . comps : if c . category == name : return c return None


def _calc_loglikelihood(self, model=None, tile=None):
        """Allows for fast local updates of log-likelihood"""
        if model is None:
            res = self.residuals
        else:
            res = model - self._data[tile.slicer]

        sig, isig = self.sigma, 1.0/self.sigma
        nlogs = -np.log(np.sqrt(2*np.pi)*sig)*res.size
        return -0.5*isig*isig*np.dot(res.flat, res.flat) + nlogs

def _calc_loglikelihood ( self , model = None , tile = None ) : if model is None : res = self . residuals else : res = model - self . _data [ tile . slicer ] sig , isig = self . sigma , 1.0 / self . sigma nlogs = - np . log ( np . sqrt ( 2 * np . pi ) * sig ) * res . size return - 0.5 * isig * isig * np . dot ( res . flat , res . flat ) + nlogs

Allows for fast local updates of log - likelihood


def update_from_model_change(self, oldmodel, newmodel, tile):
        """
        Update various internal variables from a model update from oldmodel to
        newmodel for the tile `tile`
        """
        self._loglikelihood -= self._calc_loglikelihood(oldmodel, tile=tile)
        self._loglikelihood += self._calc_loglikelihood(newmodel, tile=tile)
        self._residuals[tile.slicer] = self._data[tile.slicer] - newmodel

def update_from_model_change ( self , oldmodel , newmodel , tile ) : self . _loglikelihood -= self . _calc_loglikelihood ( oldmodel , tile = tile ) self . _loglikelihood += self . _calc_loglikelihood ( newmodel , tile = tile ) self . _residuals [ tile . slicer ] = self . _data [ tile . slicer ] - newmodel

Update various internal variables from a model update from oldmodel to newmodel for the tile tile


def set_mem_level(self, mem_level='hi'):
        """
        Sets the memory usage level of the state.

        Parameters
        ----------
        mem_level : string
            Can be set to one of:
                * hi      : all mem's are np.float64
                * med-hi  : image, platonic are float32, rest are float64
                * med     : all mem's are float32
                * med-lo  : image, platonic are float16, rest float32
                * lo      : all are float16, which is bad for accuracy.

        Notes
        -----
        Right now the PSF is not affected by the mem-level changes, which is
        OK for mem but it means that self._model, self._residuals are always
        float64, which can be a chunk of mem.
        """
        #A little thing to parse strings for convenience:
        key = ''.join([c if c in 'mlh' else '' for c in mem_level])
        if key not in ['h','mh','m','ml','m', 'l']:
            raise ValueError('mem_level must be one of hi, med-hi, med, med-lo, lo.')
        mem_levels = {  'h':     [np.float64, np.float64],
                        'mh': [np.float64, np.float32],
                        'm':   [np.float32, np.float32],
                        'ml':  [np.float32, np.float16],
                        'l':      [np.float16, np.float16]
                    }
        hi_lvl, lo_lvl = mem_levels[key]
        cat_lvls = {'obj':lo_lvl,
                    'ilm':hi_lvl,
                    'bkg':lo_lvl
                    }  #no psf...

        self.image.float_precision = hi_lvl
        self.image.image = self.image.image.astype(lo_lvl)
        self.set_image(self.image)

        for cat in cat_lvls.keys():
            obj = self.get(cat)
            #check if it's a component collection
            if hasattr(obj, 'comps'):
                for c in obj.comps:
                    c.float_precision = lo_lvl
            else:
                obj.float_precision = lo_lvl
        self._model = self._model.astype(hi_lvl)
        self._residuals = self._model.astype(hi_lvl)
        self.reset()

Sets the memory usage level of the state .


def scramble_positions(p, delete_frac=0.1):
    """randomly deletes particles and adds 1-px noise for a realistic
    initial featuring guess"""
    probs = [1-delete_frac, delete_frac]
    m = np.random.choice([True, False], p.shape[0], p=probs)
    jumble = np.random.randn(m.sum(), 3)
    return p[m] + jumble

def scramble_positions ( p , delete_frac = 0.1 ) : probs = [ 1 - delete_frac , delete_frac ] m = np . random . choice ( [ True , False ] , p . shape [ 0 ] , p = probs ) jumble = np . random . randn ( m . sum ( ) , 3 ) return p [ m ] + jumble

randomly deletes particles and adds 1 - px noise for a realistic initial featuring guess


def create_img():
    """Creates an image, as a `peri.util.Image`, which is similar
    to the image in the tutorial"""
    # 1. particles + coverslip
    rad = 0.5 * np.random.randn(POS.shape[0]) + 4.5  # 4.5 +- 0.5 px particles
    part = objs.PlatonicSpheresCollection(POS, rad, zscale=0.89)
    slab = objs.Slab(zpos=4.92, angles=(-4.7e-3, -7.3e-4))
    objects = comp.ComponentCollection([part, slab], category='obj')

    # 2. psf, ilm
    p = exactpsf.FixedSSChebLinePSF(kfki=1.07, zslab=-29.3, alpha=1.17,
            n2n1=0.98, sigkf=-0.33, zscale=0.89, laser_wavelength=0.45)
    i = ilms.BarnesStreakLegPoly2P1D(npts=(16,10,8,4), zorder=8)
    b = ilms.LegendrePoly2P1D(order=(7,2,2), category='bkg')
    off = comp.GlobalScalar(name='offset', value=-2.11)
    mdl = models.ConfocalImageModel()
    st = states.ImageState(util.NullImage(shape=[48,64,64]),
            [objects, p, i, b, off], mdl=mdl, model_as_data=True)
    b.update(b.params, BKGVALS)
    i.update(i.params, ILMVALS)
    im = st.model + np.random.randn(*st.model.shape) * 0.03
    return util.Image(im)

def
create_img
(
)
:
# 1. particles + coverslip
rad
=
0.5
*
np
.
random
.
randn
(
POS
.
shape
[
0
]
)
+
4.5
# 4.5 +- 0.5 px particles
part
=
objs
.
PlatonicSpheresCollection
(
POS
,
rad
,
zscale
=
0.89
)
slab
=
objs
.
Slab
(
zpos
=
4.92
,
angles
=
(
4.7e-3
,
7.3e-4
)
)
objects
=
comp
.
ComponentCollection
(
[
part
,
slab
]
,
category
=
'obj'
)
# 2. psf, ilm
p
=
exactpsf
.
FixedSSChebLinePSF
(
kfki
=
1.07
,
zslab
=
29.3
,
alpha
=
1.17
,
n2n1
=
0.98
,
sigkf
=
0.33
,
zscale
=
0.89
,
laser_wavelength
=
0.45
)
i
=
ilms
.
BarnesStreakLegPoly2P1D
(
npts
=
(
16
,
10
,
8
,
4
)
,
zorder
=
8
)
b
=
ilms
.
LegendrePoly2P1D
(
order
=
(
7
,
2
,
2
)
,
category
=
'bkg'
)
off
=
comp
.
GlobalScalar
(
name
=
'offset'
,
value
=
2.11
)
mdl
=
models
.
ConfocalImageModel
(
)
st
=
states
.
ImageState
(
util
.
NullImage
(
shape
=
[
48
,
64
,
64
]
)
,
[
objects
,
p
,
i
,
b
,
off
]
,
mdl
=
mdl
,
model_as_data
=
True
)
b
.
update
(
b
.
params
,
BKGVALS
)
i
.
update
(
i
.
params
,
ILMVALS
)
im
=
st
.
model
+
np
.
random
.
randn
(
*
st
.
model
.
shape
)
*
0.03
return
util
.
Image
(
im
)

Creates an image as a peri . util . Image which is similar to the image in the tutorial


def get_values(self, params):
        """
        Get the value of a list or single parameter.

        Parameters
        ----------
        params : string, list of string
            name of parameters which to retrieve
        """
        return util.delistify(
            [self.param_dict[p] for p in util.listify(params)], params
        )

def get_values ( self , params ) : return util . delistify ( [ self . param_dict [ p ] for p in util . listify ( params ) ] , params )

Get the value of a list or single parameter .


def set_shape(self, shape, inner):
        """
        Set the overall shape of the calculation area. The total shape of that
        the calculation can possibly occupy, in pixels. The second, inner, is
        the region of interest within the image.
        """
        if self.shape != shape or self.inner != inner:
            self.shape = shape
            self.inner = inner
            self.initialize()

def set_shape ( self , shape , inner ) : if self . shape != shape or self . inner != inner : self . shape = shape self . inner = inner self . initialize ( )

Set the overall shape of the calculation area . The total shape of that the calculation can possibly occupy in pixels . The second inner is the region of interest within the image .


def trigger_update(self, params, values):
        """ Notify parent of a parameter change """
        if self._parent:
            self._parent.trigger_update(params, values)
        else:
            self.update(params, values)

def trigger_update ( self , params , values ) : if self . _parent : self . _parent . trigger_update ( params , values ) else : self . update ( params , values )


def get(self):
        """ Combine the fields from all components """
        fields = [c.get() for c in self.comps]
        return self.field_reduce_func(fields)

def get ( self ) : fields = [ c . get ( ) for c in self . comps ] return self . field_reduce_func ( fields )


def set_shape(self, shape, inner):
        """ Set the shape for all components """
        for c in self.comps:
            c.set_shape(shape, inner)

def set_shape ( self , shape , inner ) : for c in self . comps : c . set_shape ( shape , inner )


def sync_params(self):
        """ Ensure that shared parameters are the same value everywhere """
        def _normalize(comps, param):
            vals = [c.get_values(param) for c in comps]
            diff = any([vals[i] != vals[i+1] for i in range(len(vals)-1)])

            if diff:
                for c in comps:
                    c.set_values(param, vals[0])

        for param, comps in iteritems(self.lmap):
            if isinstance(comps, list) and len(comps) > 1:
                _normalize(comps, param)

def sync_params ( self ) : def _normalize ( comps , param ) : vals = [ c . get_values ( param ) for c in comps ] diff = any ( [ vals [ i ] != vals [ i + 1 ] for i in range ( len ( vals ) - 1 ) ] ) if diff : for c in comps : c . set_values ( param , vals [ 0 ] ) for param , comps in iteritems ( self . lmap ) : if isinstance ( comps , list ) and len ( comps ) > 1 : _normalize ( comps , param )