def ratio(self, internal_standard=None):
        """
        Divide all analytes by a specified internal_standard analyte.

        Parameters
        ----------
        internal_standard : str
            The analyte used as the internal_standard.

        Returns
        -------
        None
        """
        if internal_standard is not None:
            self.internal_standard = internal_standard

        self.data['ratios'] = Bunch()
        for a in self.analytes:
            self.data['ratios'][a] = (self.data['bkgsub'][a] /
                                      self.data['bkgsub'][self.internal_standard])
        self.setfocus('ratios')
        return

def ratio ( self , internal_standard = None ) : if internal_standard is not None : self . internal_standard = internal_standard self . data [ 'ratios' ] = Bunch ( ) for a in self . analytes : self . data [ 'ratios' ] [ a ] = ( self . data [ 'bkgsub' ] [ a ] / self . data [ 'bkgsub' ] [ self . internal_standard ] ) self . setfocus ( 'ratios' ) return

Divide all analytes by a specified internal_standard analyte .


def calibrate(self, calib_ps, analytes=None):
        """
        Apply calibration to data.

        The `calib_dict` must be calculated at the `analyse` level,
        and passed to this calibrate function.

        Parameters
        ----------
        calib_dict : dict
            A dict of calibration values to apply to each analyte.

        Returns
        -------
        None
        """
        # can have calibration function stored in self and pass *coefs?
        if analytes is None:
            analytes = self.analytes

        if 'calibrated' not in self.data.keys():
            self.data['calibrated'] = Bunch()

        for a in analytes:
            m = calib_ps[a]['m'].new(self.uTime)

            if 'c' in calib_ps[a]:
                c = calib_ps[a]['c'].new(self.uTime)
            else:
                c = 0

            self.data['calibrated'][a] = self.data['ratios'][a] * m + c

        if self.internal_standard not in analytes:
            self.data['calibrated'][self.internal_standard] = \
                np.empty(len(self.data['ratios'][self.internal_standard]))

        self.setfocus('calibrated')
        return


def sample_stats(self, analytes=None, filt=True,
                     stat_fns={},
                     eachtrace=True):
        """
        Calculate sample statistics

        Returns samples, analytes, and arrays of statistics
        of shape (samples, analytes). Statistics are calculated
        from the 'focus' data variable, so output depends on how
        the data have been processed.

        Parameters
        ----------
        analytes : array_like
            List of analytes to calculate the statistic on
        filt : bool or str
            The filter to apply to the data when calculating sample statistics.
                bool: True applies filter specified in filt.switches.
                str: logical string specifying a partucular filter
        stat_fns : dict
            Dict of {name: function} pairs. Functions that take a single
            array_like input, and return a single statistic. Function should
            be able to cope with NaN values.
        eachtrace : bool
            True: per - ablation statistics
            False: whole sample statistics

        Returns
        -------
        None
        """
        if analytes is None:
                analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        self.stats = Bunch()
        self.stats['analytes'] = analytes

        with warnings.catch_warnings():
            warnings.simplefilter("ignore", category=RuntimeWarning)
            for n, f in stat_fns.items():
                self.stats[n] = []
                for a in analytes:
                    ind = self.filt.grab_filt(filt, a)
                    dat = nominal_values(self.focus[a])
                    if eachtrace:
                        sts = []
                        for t in np.arange(self.n) + 1:
                            sts.append(f(dat[ind & (self.ns == t)]))
                        self.stats[n].append(sts)
                    else:
                        self.stats[n].append(f(dat[ind]))
                self.stats[n] = np.array(self.stats[n])
        return

def sample_stats ( self , analytes = None , filt = True , stat_fns = { } , eachtrace = True ) : if analytes is None : analytes = self . analytes elif isinstance ( analytes , str ) : analytes = [ analytes ] self . stats = Bunch ( ) self . stats [ 'analytes' ] = analytes with warnings . catch_warnings ( ) : warnings . simplefilter ( "ignore" , category = RuntimeWarning ) for n , f in stat_fns . items ( ) : self . stats [ n ] = [ ] for a in analytes : ind = self . filt . grab_filt ( filt , a ) dat = nominal_values ( self . focus [ a ] ) if eachtrace : sts = [ ] for t in np . arange ( self . n ) + 1 : sts . append ( f ( dat [ ind & ( self . ns == t ) ] ) ) self . stats [ n ] . append ( sts ) else : self . stats [ n ] . append ( f ( dat [ ind ] ) ) self . stats [ n ] = np . array ( self . stats [ n ] ) return


def ablation_times(self):
        """
        Function for calculating the ablation time for each
        ablation.

        Returns
        -------
            dict of times for each ablation.
        """
        ats = {}
        for n in np.arange(self.n) + 1:
            t = self.Time[self.ns == n]
            ats[n - 1] = t.max() - t.min()
        return ats

def ablation_times ( self ) : ats = { } for n in np . arange ( self . n ) + 1 : t = self . Time [ self . ns == n ] ats [ n - 1 ] = t . max ( ) - t . min ( ) return ats

Function for calculating the ablation time for each ablation .


def filter_threshold(self, analyte, threshold):
        """
        Apply threshold filter.

        Generates threshold filters for the given analytes above and below
        the specified threshold.

        Two filters are created with prefixes '_above' and '_below'.
            '_above' keeps all the data above the threshold.
            '_below' keeps all the data below the threshold.

        i.e. to select data below the threshold value, you should turn the
        '_above' filter off.

        Parameters
        ----------
        analyte : TYPE
            Description of `analyte`.
        threshold : TYPE
            Description of `threshold`.

        Returns
        -------
        None
        """
        params = locals()
        del(params['self'])

        # generate filter
        below, above = filters.threshold(self.focus[analyte], threshold)

        setn = self.filt.maxset + 1

        self.filt.add(analyte + '_thresh_below',
                        below,
                        'Keep below {:.3e} '.format(threshold) + analyte,
                        params, setn=setn)
        self.filt.add(analyte + '_thresh_above',
                        above,
                        'Keep above {:.3e} '.format(threshold) + analyte,
                        params, setn=setn)

def
filter_threshold
(
self
,
analyte
,
threshold
)
:
params
=
locals
(
)
del
(
params
[
'self'
]
)
# generate filter
below
,
above
=
filters
.
threshold
(
self
.
focus
[
analyte
]
,
threshold
)
setn
=
self
.
filt
.
maxset
+
1
self
.
filt
.
add
(
analyte
+
'_thresh_below'
,
below
,
'Keep below {:.3e} '
.
format
(
threshold
)
+
analyte
,
params
,
setn
=
setn
)
self
.
filt
.
add
(
analyte
+
'_thresh_above'
,
above
,
'Keep above {:.3e} '
.
format
(
threshold
)
+
analyte
,
params
,
setn
=
setn
)


def filter_gradient_threshold(self, analyte, win, threshold, recalc=True):
        """
        Apply gradient threshold filter.

        Generates threshold filters for the given analytes above and below
        the specified threshold.

        Two filters are created with prefixes '_above' and '_below'.
            '_above' keeps all the data above the threshold.
            '_below' keeps all the data below the threshold.

        i.e. to select data below the threshold value, you should turn the
        '_above' filter off.

        Parameters
        ----------
        analyte : str
            Description of `analyte`.
        threshold : float
            Description of `threshold`.
        win : int
            Window used to calculate gradients (n points)
        recalc : bool
            Whether or not to re-calculate the gradients.

        Returns
        -------
        None
        """
        params = locals()
        del(params['self'])

        # calculate absolute gradient
        if recalc or not self.grads_calced:
            self.grads = calc_grads(self.Time, self.focus,
                                    [analyte], win)
            self.grads_calced = True

        below, above = filters.threshold(abs(self.grads[analyte]), threshold)

        setn = self.filt.maxset + 1

        self.filt.add(analyte + '_gthresh_below',
                        below,
                        'Keep gradient below {:.3e} '.format(threshold) + analyte,
                        params, setn=setn)
        self.filt.add(analyte + '_gthresh_above',
                        above,
                        'Keep gradient above {:.3e} '.format(threshold) + analyte,
                        params, setn=setn)

def
filter_gradient_threshold
(
self
,
analyte
,
win
,
threshold
,
recalc
=
True
)
:
params
=
locals
(
)
del
(
params
[
'self'
]
)
# calculate absolute gradient
if
recalc
or
not
self
.
grads_calced
:
self
.
grads
=
calc_grads
(
self
.
Time
,
self
.
focus
,
[
analyte
]
,
win
)
self
.
grads_calced
=
True
below
,
above
=
filters
.
threshold
(
abs
(
self
.
grads
[
analyte
]
)
,
threshold
)
setn
=
self
.
filt
.
maxset
+
1
self
.
filt
.
add
(
analyte
+
'_gthresh_below'
,
below
,
'Keep gradient below {:.3e} '
.
format
(
threshold
)
+
analyte
,
params
,
setn
=
setn
)
self
.
filt
.
add
(
analyte
+
'_gthresh_above'
,
above
,
'Keep gradient above {:.3e} '
.
format
(
threshold
)
+
analyte
,
params
,
setn
=
setn
)


def calc_correlation(self, x_analyte, y_analyte, window=15, filt=True, recalc=True):
        """
        Calculate local correlation between two analytes.

        Parameters
        ----------
        x_analyte, y_analyte : str
            The names of the x and y analytes to correlate.
        window : int, None
            The rolling window used when calculating the correlation.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        recalc : bool
            If True, the correlation is re-calculated, even if it is already present.

        Returns
        -------
        None
        """
        label = '{:}_{:}_{:.0f}'.format(x_analyte, y_analyte, window)

        if label in self.correlations and not recalc:
            return

        # make window odd
        if window % 2 != 1:
            window += 1

        # get filter
        ind = self.filt.grab_filt(filt, [x_analyte, y_analyte])

        x = nominal_values(self.focus[x_analyte])
        x[~ind] = np.nan
        xr = rolling_window(x, window, pad=np.nan)

        y = nominal_values(self.focus[y_analyte])
        y[~ind] = np.nan
        yr = rolling_window(y, window, pad=np.nan)

        r, p = zip(*map(nan_pearsonr, xr, yr))

        r = np.array(r)
        p = np.array(p)

        # save correlation info

        self.correlations[label] = r, p
        return

def
calc_correlation
(
self
,
x_analyte
,
y_analyte
,
window
=
15
,
filt
=
True
,
recalc
=
True
)
:
label
=
'{:}_{:}_{:.0f}'
.
format
(
x_analyte
,
y_analyte
,
window
)
if
label
in
self
.
correlations
and
not
recalc
:
return
# make window odd
if
window
%
2
!=
1
:
window
+=
1
# get filter
ind
=
self
.
filt
.
grab_filt
(
filt
,
[
x_analyte
,
y_analyte
]
)
x
=
nominal_values
(
self
.
focus
[
x_analyte
]
)
x
[
~
ind
]
=
np
.
xr
=
rolling_window
(
x
,
window
,
pad
=
np
.
)
y
=
nominal_values
(
self
.
focus
[
y_analyte
]
)
y
[
~
ind
]
=
np
.
yr
=
rolling_window
(
y
,
window
,
pad
=
np
.
)
r
,
p
=
zip
(
*
map
(
nan_pearsonr
,
xr
,
yr
)
)
r
=
np
.
array
(
r
)
p
=
np
.
array
(
p
)
# save correlation info
self
.
correlations
[
label
]
=
r
,
p
return

Calculate local correlation between two analytes .


def filter_correlation(self, x_analyte, y_analyte, window=15,
                           r_threshold=0.9, p_threshold=0.05, filt=True, recalc=False):
        """
        Calculate correlation filter.

        Parameters
        ----------
        x_analyte, y_analyte : str
            The names of the x and y analytes to correlate.
        window : int, None
            The rolling window used when calculating the correlation.
        r_threshold : float
            The correlation index above which to exclude data.
            Note: the absolute pearson R value is considered, so
            negative correlations below -`r_threshold` will also
            be excluded.
        p_threshold : float
            The significant level below which data are excluded.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        recalc : bool
            If True, the correlation is re-calculated, even if it is already present.

        Returns
        -------
        None
        """
        # make window odd
        if window % 2 != 1:
            window += 1

        params = locals()
        del(params['self'])

        setn = self.filt.maxset + 1

        label = '{:}_{:}_{:.0f}'.format(x_analyte, y_analyte, window)

        self.calc_correlation(x_analyte, y_analyte, window, filt, recalc)
        r, p = self.correlations[label]

        cfilt = (abs(r) > r_threshold) & (p < p_threshold)
        cfilt = ~cfilt

        name = x_analyte + '_' + y_analyte + '_corr'

        self.filt.add(name=name,
                      filt=cfilt,
                      info=(x_analyte + ' vs. ' + y_analyte +
                            ' correlation filter.'),
                      params=params, setn=setn)
        self.filt.off(filt=name)
        self.filt.on(analyte=y_analyte, filt=name)

        return

def
filter_correlation
(
self
,
x_analyte
,
y_analyte
,
window
=
15
,
r_threshold
=
0.9
,
p_threshold
=
0.05
,
filt
=
True
,
recalc
=
False
)
:
# make window odd
if
window
%
2
!=
1
:
window
+=
1
params
=
locals
(
)
del
(
params
[
'self'
]
)
setn
=
self
.
filt
.
maxset
+
1
label
=
'{:}_{:}_{:.0f}'
.
format
(
x_analyte
,
y_analyte
,
window
)
self
.
calc_correlation
(
x_analyte
,
y_analyte
,
window
,
filt
,
recalc
)
r
,
p
=
self
.
correlations
[
label
]
cfilt
=
(
abs
(
r
)
>
r_threshold
)
&
(
p
<
p_threshold
)
cfilt
=
~
cfilt
name
=
x_analyte
+
'_'
+
y_analyte
+
'_corr'
self
.
filt
.
add
(
name
=
name
,
filt
=
cfilt
,
info
=
(
x_analyte
+
' vs. '
+
y_analyte
+
' correlation filter.'
)
,
params
=
params
,
setn
=
setn
)
self
.
filt
.
off
(
filt
=
name
)
self
.
filt
.
on
(
analyte
=
y_analyte
,
filt
=
name
)
return


def filter_new(self, name, filt_str):
        """
        Make new filter from combination of other filters.

        Parameters
        ----------
        name : str
            The name of the new filter. Should be unique.
        filt_str : str
            A logical combination of partial strings which will create
            the new filter. For example, 'Albelow & Mnbelow' will combine
            all filters that partially match 'Albelow' with those that
            partially match 'Mnbelow' using the 'AND' logical operator.

        Returns
        -------
        None
        """
        filt = self.filt.grab_filt(filt=filt_str)
        self.filt.add(name, filt, info=filt_str)
        return

def filter_new ( self , name , filt_str ) : filt = self . filt . grab_filt ( filt = filt_str ) self . filt . add ( name , filt , info = filt_str ) return

Make new filter from combination of other filters .


def get_params(self):
        """
        Returns paramters used to process data.

        Returns
        -------
        dict
            dict of analysis parameters
        """
        outputs = ['sample',
                   'ratio_params',
                   'despike_params',
                   'autorange_params',
                   'bkgcorrect_params']

        out = {}
        for o in outputs:
            out[o] = getattr(self, o)

        out['filter_params'] = self.filt.params
        out['filter_sequence'] = self.filt.sequence
        out['filter_used'] = self.filt.make_keydict()

        return out

def get_params ( self ) : outputs = [ 'sample' , 'ratio_params' , 'despike_params' , 'autorange_params' , 'bkgcorrect_params' ] out = { } for o in outputs : out [ o ] = getattr ( self , o ) out [ 'filter_params' ] = self . filt . params out [ 'filter_sequence' ] = self . filt . sequence out [ 'filter_used' ] = self . filt . make_keydict ( ) return out

Returns paramters used to process data .


def histograms(dat, keys=None, bins=25, logy=False, cmap=None, ncol=4):
    """
    Plot histograms of all items in dat.

    Parameters
    ----------
    dat : dict
        Data in {key: array} pairs.
    keys : arra-like
        The keys in dat that you want to plot. If None,
        all are plotted.
    bins : int
        The number of bins in each histogram (default = 25)
    logy : bool
        If true, y axis is a log scale.
    cmap : dict
        The colours that the different items should be. If None,
        all are grey.

    Returns
    -------
    fig, axes
    """
    if keys is None:
        keys = dat.keys()

    ncol = int(ncol)
    nrow = calc_nrow(len(keys), ncol)

    fig, axs = plt.subplots(nrow, 4, figsize=[ncol * 2, nrow * 2])

    pn = 0
    for k, ax in zip(keys, axs.flat):
        tmp = nominal_values(dat[k])
        x = tmp[~np.isnan(tmp)]

        if cmap is not None:
            c = cmap[k]
        else:
            c = (0, 0, 0, 0.5)
        ax.hist(x, bins=bins, color=c)

        if logy:
            ax.set_yscale('log')
            ylab = '$log_{10}(n)$'
        else:
            ylab = 'n'

        ax.set_ylim(1, ax.get_ylim()[1])

        if ax.is_first_col():
            ax.set_ylabel(ylab)

        ax.set_yticklabels([])

        ax.text(.95, .95, k, ha='right', va='top', transform=ax.transAxes)

        pn += 1

    for ax in axs.flat[pn:]:
        ax.set_visible(False)
    fig.tight_layout()

    return fig, axs

def histograms ( dat , keys = None , bins = 25 , logy = False , cmap = None , ncol = 4 ) : if keys is None : keys = dat . keys ( ) ncol = int ( ncol ) nrow = calc_nrow ( len ( keys ) , ncol ) fig , axs = plt . subplots ( nrow , 4 , figsize = [ ncol * 2 , nrow * 2 ] ) pn = 0 for k , ax in zip ( keys , axs . flat ) : tmp = nominal_values ( dat [ k ] ) x = tmp [ ~ np . isnan ( tmp ) ] if cmap is not None : c = cmap [ k ] else : c = ( 0 , 0 , 0 , 0.5 ) ax . hist ( x , bins = bins , color = c ) if logy : ax . set_yscale ( 'log' ) ylab = '$log_{10}(n)$' else : ylab = 'n' ax . set_ylim ( 1 , ax . get_ylim ( ) [ 1 ] ) if ax . is_first_col ( ) : ax . set_ylabel ( ylab ) ax . set_yticklabels ( [ ] ) ax . text ( .95 , .95 , k , ha = 'right' , va = 'top' , transform = ax . transAxes ) pn += 1 for ax in axs . flat [ pn : ] : ax . set_visible ( False ) fig . tight_layout ( ) return fig , axs


def summary_stats(x, y, nm=None):
    """
    Compute summary statistics for paired x, y data.

    Tests
    -----

    Parameters
    ----------
    x, y : array-like
        Data to compare
    nm : str (optional)
        Index value of created dataframe.

    Returns
    -------
    pandas dataframe of statistics.
    """
    # create datafrane for results
    if isinstance(nm, str):
        nm = [nm]
    # cols = pd.MultiIndex.from_arrays([['', 'Pairwise', 'Pairwise', cat, cat, cat, cat],
    #                                   ['N', 'W', 'p', 'Median', 'IQR', 'W', 'p']])
#     cols = ['Median', 'IQR', 'CI95', 'L95', 'LQ', 'UQ', 'U95', 'N',
#             'Wilcoxon_stat', 'Wilcoxon_p',
#             'KS_stat', 'KS_p',
#             'LR_slope', 'LR_intercept', 'LR_slope_tvalue', 'LR_intercept_tvalue', 'LR_slope_p', 'LR_intercept_p', 'LR_R2adj']
#     out = pd.DataFrame(index=nm, columns=cols)

    cols = pd.MultiIndex.from_tuples([('Residual Summary', 'N'),
                                      ('Residual Summary', 'Median'),
                                      ('Residual Summary', 'LQ'),
                                      ('Residual Summary', 'IQR'),
                                      ('Residual Summary', 'UQ'),
                                      ('Residual Regression', 'Slope'),
                                      ('Residual Regression', 'Slope t'),
                                      ('Residual Regression', 'Slope p'),
                                      ('Residual Regression', 'Intercept'),
                                      ('Residual Regression', 'Intercept t'),
                                      ('Residual Regression', 'Intercept p'),
                                      ('Residual Regression', 'R2'),
                                      ('Kolmogorov-Smirnov', 'KS'),
                                      ('Kolmogorov-Smirnov', 'p')])

    out = pd.DataFrame(index=nm, columns=cols)

    # remove nan values
    ind = ~(np.isnan(x) | np.isnan(y))
    x = x[ind]
    y = y[ind]

    # calculate residuals
    r = y - x

    # summary statistics
    cat = 'Residual Summary'
    out.loc[:, (cat, 'N')] = len(x)
    out.loc[:, (cat, 'Median')] = np.median(r)
    out.loc[:, [(cat, 'LQ'), (cat, 'UQ')]] = np.percentile(r, [25, 75])
    out.loc[:, (cat, 'IQR')] = out.loc[:, (cat, 'UQ')] - out.loc[:, (cat, 'LQ')]

    # non-paired test for same distribution
    cat = 'Kolmogorov-Smirnov'
    ks = stats.ks_2samp(x, y)
    out.loc[:, (cat, 'KS')] = ks.statistic
    out.loc[:, (cat, 'p')] = ks.pvalue

    # regression analysis of residuals - slope should be 0, intercept should be 0
    cat = 'Residual Regression'
    X = sm.add_constant(x)
    reg = sm.OLS(r, X, missing='drop')
    fit = reg.fit()

    out.loc[:, [(cat, 'Intercept'), (cat, 'Slope')]] = fit.params
    out.loc[:, [(cat, 'Intercept t'), (cat, 'Slope t')]] = fit.tvalues
    out.loc[:, (cat, 'R2')] = fit.rsquared
    out.loc[:, [(cat, 'Intercept p'), (cat, 'Slope p')]] = fit.pvalues

    return out

#             'Wilcoxon_stat', 'Wilcoxon_p',

Compute summary statistics for paired x y data .


def load_reference_data(name=None):
    """
    Fetch LAtools reference data from online repository.

    Parameters
    ----------
    name : str<
        Which data to download. Can be one of 'culture_reference',
        'culture_test', 'downcore_reference', 'downcore_test', 'iolite_reference'
        or 'zircon_reference'.
        If None, all are downloaded and returned as a dict.

    Returns
    -------
    pandas.DataFrame or dict.
    """
    base_url = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vQJfCeuqrtFFMAeSpA9rguzLAo9OVuw50AHhAULuqjMJzbd3h46PK1KjF69YiJAeNAAjjMDkJK7wMpG/pub?gid={:}&single=true&output=csv'
    gids = {'culture_reference': '0',
            'culture_test': '1170065442',
            'downcore_reference': '190752797',
            'downcore_test': '721359794',
            'iolite_reference': '483581945',
            'zircon_reference': '1355554964'}

    if name is None:
        out = {}
        for nm, gid in gids.items():
            url = base_url.format(gid)
            tmp = pd.read_csv(url, header=[0], index_col=[0, 1])
            tmp.index.names = ['sample', 'rep']
            tmp.columns.names = ['analyte']
            tmp.sort_index(1, inplace=True)
            out[nm] = tmp
    else:
        gid = gids[name]
        url = base_url.format(gid)
        out = pd.read_csv(url, index_col=[0, 1])
        out.columns.names = ['analyte']
        out.sort_index(1, inplace=True)
    return out

Fetch LAtools reference data from online repository .


def lookup(self, TC: type, G: type) -> Optional[TypeClass]:
        ''' Find an instance of the type class `TC` for type `G`.
        Iterates `G`'s parent classes, looking up instances for each,
        checking whether the instance is a subclass of the target type
        class `TC`.
        '''
        if isinstance(G, str):
            raise ImplicitNotFound(TC, G, f'{G} is a string annotation')
        if not isinstance(G, (type, TypeVar, _GenericAlias)):
            raise ImplicitNotFound(TC, G, f'{G} is neither type, _GenericAlias nor TypeVar: {type(G)}')
        match = lambda a: self._lookup_type(TC, a)
        def attach_type(tc: TypeClass) -> TypeClass:
            setattr(tc, 'tpe', G)
            return tc
        scrutinee = (
            (
                unbounded_typevar(TC, G)
                if G.__bound__ is None else
                G.__bound__
            )
            if isinstance(G, TypeVar) else
            G
        )
        safe_mro = lambda t: getattr(t, '__mro__', (t,))
        target = scrutinee.__origin__ if isinstance(scrutinee, _GenericAlias) else scrutinee
        mro = safe_mro(target)
        return next((attach_type(a) for a in map(match, mro) if a is not None), None)

Find an instance of the type class TC for type G . Iterates G s parent classes looking up instances for each checking whether the instance is a subclass of the target type class TC .


def elements(all_isotopes=True):
    """
    Loads a DataFrame of all elements and isotopes.

    Scraped from https://www.webelements.com/

    Returns
    -------
    pandas DataFrame with columns (element, atomic_number, isotope, atomic_weight, percent)
    """
    el = pd.read_pickle(pkgrs.resource_filename('latools', 'resources/elements.pkl'))
    if all_isotopes:
        return el.set_index('element')
    else:
        def wmean(g):
            return (g.atomic_weight * g.percent).sum() / 100
        iel = el.groupby('element').apply(wmean)
        iel.name = 'atomic_weight'
        return iel

def elements ( all_isotopes = True ) : el = pd . read_pickle ( pkgrs . resource_filename ( 'latools' , 'resources/elements.pkl' ) ) if all_isotopes : return el . set_index ( 'element' ) else : def wmean ( g ) : return ( g . atomic_weight * g . percent ) . sum ( ) / 100 iel = el . groupby ( 'element' ) . apply ( wmean ) iel . name = 'atomic_weight' return iel

Loads a DataFrame of all elements and isotopes .


def calc_M(molecule):
    """
    Returns molecular weight of molecule.

    Where molecule is in standard chemical notation,
    e.g. 'CO2', 'HCO3' or B(OH)4

    Returns
    -------
    molecular_weight : float
    """

    # load periodic table
    els = elements()

    # define regexs
    parens = re.compile('\(([A-z0-9]+)\)([0-9]+)?')
    stoich = re.compile('([A-Z][a-z]?)([0-9]+)?')

    ps = parens.findall(molecule)  # find subgroups in parentheses
    rem = parens.sub('', molecule)  # get remainder

    m = 0
    # deal with sub-groups
    if len(ps) > 0:
        for sub, ns in ps:
            ms = 0
            for e, n in stoich.findall(sub):
                me = (els.loc[e, 'atomic_weight'] *
                      els.loc[e, 'percent'] / 100).sum()
                if n == '':
                    n = 1
                else:
                    n = int(n)
                ms += me * n
            if ns == '':
                ns = 1
            else:
                ns = int(ns)
            m += ms * ns
    # deal with remainder
    for e, n in stoich.findall(rem):
        me = (els.loc[e, 'atomic_weight'] *
              els.loc[e, 'percent'] / 100).sum()
        if n == '':
            n = 1
        else:
            n = int(n)
        m += me * n
    return m

def
calc_M
(
molecule
)
:
# load periodic table
els
=
elements
(
)
# define regexs
parens
=
re
.
compile
(
'\(([A-z0-9]+)\)([0-9]+)?'
)
stoich
=
re
.
compile
(
'([A-Z][a-z]?)([0-9]+)?'
)
ps
=
parens
.
findall
(
molecule
)
# find subgroups in parentheses
rem
=
parens
.
sub
(
''
,
molecule
)
# get remainder
m
=
0
# deal with sub-groups
if
len
(
ps
)
>
0
:
for
sub
,
ns
in
ps
:
ms
=
0
for
e
,
n
in
stoich
.
findall
(
sub
)
:
me
=
(
els
.
loc
[
e
,
'atomic_weight'
]
*
els
.
loc
[
e
,
'percent'
]
/
100
)
.
sum
(
)
if
n
==
''
:
n
=
1
else
:
n
=
int
(
n
)
ms
+=
me
*
n
if
ns
==
''
:
ns
=
1
else
:
ns
=
int
(
ns
)
m
+=
ms
*
ns
# deal with remainder
for
e
,
n
in
stoich
.
findall
(
rem
)
:
me
=
(
els
.
loc
[
e
,
'atomic_weight'
]
*
els
.
loc
[
e
,
'percent'
]
/
100
)
.
sum
(
)
if
n
==
''
:
n
=
1
else
:
n
=
int
(
n
)
m
+=
me
*
n
return
m


def gen_keywords(*args: Union[ANSIColors, ANSIStyles], **kwargs: Union[ANSIColors, ANSIStyles]) -> tuple:
    '''generate single escape sequence mapping.'''
    fields: tuple = tuple()
    values: tuple = tuple()
    for tpl in args:
        fields += tpl._fields
        values += tpl
    for prefix, tpl in kwargs.items():
        fields += tuple(map(lambda x: '_'.join([prefix, x]), tpl._fields))
        values += tpl
    return namedtuple('ANSISequences', fields)(*values)

def gen_keywords ( * args : Union [ ANSIColors , ANSIStyles ] , * * kwargs : Union [ ANSIColors , ANSIStyles ] ) -> tuple : fields : tuple = tuple ( ) values : tuple = tuple ( ) for tpl in args : fields += tpl . _fields values += tpl for prefix , tpl in kwargs . items ( ) : fields += tuple ( map ( lambda x : '_' . join ( [ prefix , x ] ) , tpl . _fields ) ) values += tpl return namedtuple ( 'ANSISequences' , fields ) ( * values )

generate single escape sequence mapping .


def annihilate(predicate: tuple, stack: tuple) -> tuple:
    '''Squash and reduce the input stack.
    Removes the elements of input that match predicate and only keeps the last
    match at the end of the stack.
    '''
    extra = tuple(filter(lambda x: x not in predicate, stack))
    head = reduce(lambda x, y: y if y in predicate else x, stack, None)
    return extra + (head,) if head else extra

def annihilate ( predicate : tuple , stack : tuple ) -> tuple : extra = tuple ( filter ( lambda x : x not in predicate , stack ) ) head = reduce ( lambda x , y : y if y in predicate else x , stack , None ) return extra + ( head , ) if head else extra

Squash and reduce the input stack . Removes the elements of input that match predicate and only keeps the last match at the end of the stack .


def dedup(stack: tuple) -> tuple:
    '''Remove duplicates from the stack in first-seen order.'''
    # Initializes with an accumulator and then reduces the stack with first match
    # deduplication.
    reducer = lambda x, y: x if y in x else x + (y,)
    return reduce(reducer, stack, tuple())

Remove duplicates from the stack in first - seen order .


def gauss_weighted_stats(x, yarray, x_new, fwhm):
    """
    Calculate gaussian weigted moving mean, SD and SE.

    Parameters
    ----------
    x : array-like
        The independent variable
    yarray : (n,m) array
        Where n = x.size, and m is the number of
        dependent variables to smooth.
    x_new : array-like
        The new x-scale to interpolate the data
    fwhm : int
        FWHM of the gaussian kernel.

    Returns
    -------
    (mean, std, se) : tuple
    """
    sigma = fwhm / (2 * np.sqrt(2 * np.log(2)))

    # create empty mask array
    mask = np.zeros((x.size, yarray.shape[1], x_new.size))
    # fill mask
    for i, xni in enumerate(x_new):
        mask[:, :, i] = gauss(x[:, np.newaxis], 1, xni, sigma)
    # normalise mask
    nmask = mask / mask.sum(0)  # sum of each gaussian = 1

    # calculate moving average
    av = (nmask * yarray[:, :, np.newaxis]).sum(0)  # apply mask to data
    # sum along xn axis to get means

    # calculate moving sd
    diff = np.power(av - yarray[:, :, np.newaxis], 2)
    std = np.sqrt((diff * nmask).sum(0))
    # sqrt of weighted average of data-mean

    # calculate moving se
    se = std / np.sqrt(mask.sum(0))
    # max amplitude of weights is 1, so sum of weights scales
    # a fn of how many points are nearby. Use this as 'n' in
    # SE calculation.

    return av, std, se

Calculate gaussian weigted moving mean SD and SE .


def gauss(x, *p):
    """ Gaussian function.

    Parameters
    ----------
    x : array_like
        Independent variable.
    *p : parameters unpacked to A, mu, sigma
        A = amplitude, mu = centre, sigma = width

    Return
    ------
    array_like
        gaussian descriped by *p.
    """
    A, mu, sigma = p
    return A * np.exp(-0.5 * (-mu + x)**2 / sigma**2)

def gauss ( x , * p ) : A , mu , sigma = p return A * np . exp ( - 0.5 * ( - mu + x ) ** 2 / sigma ** 2 )


def stderr(a):
    """
    Calculate the standard error of a.
    """
    return np.nanstd(a) / np.sqrt(sum(np.isfinite(a)))

def stderr ( a ) : return np . nanstd ( a ) / np . sqrt ( sum ( np . isfinite ( a ) ) )


def _get_samples(self, subset=None):
        """
        Helper function to get sample names from subset.

        Parameters
        ----------
        subset : str
            Subset name. If None, returns all samples.

        Returns
        -------
        List of sample names
        """
        if subset is None:
            samples = self.subsets['All_Samples']
        else:
            try:
                samples = self.subsets[subset]
            except KeyError:
                raise KeyError(("Subset '{:s}' does not ".format(subset) +
                                "exist.\nUse 'make_subset' to create a" +
                                "subset."))
        return samples

def
_get_samples
(
self
,
subset
=
)
:
if
subset
is
:
samples
=
self
.
subsets
[
'All_Samples'
]
else
:
try
:
samples
=
self
.
subsets
[
subset
]
except
KeyError
:
raise
KeyError
(
(
"Subset '{:s}' does not "
.
format
(
subset
)
+
"exist.\nUse 'make_subset' to create a"
+
"subset."
)
)
return
samples

Helper function to get sample names from subset .


def despike(self, expdecay_despiker=False, exponent=None,
                noise_despiker=True, win=3, nlim=12., exponentplot=False,
                maxiter=4, autorange_kwargs={}, focus_stage='rawdata'):
        """
        Despikes data with exponential decay and noise filters.

        Parameters
        ----------
        expdecay_despiker : bool
            Whether or not to apply the exponential decay filter.
        exponent : None or float
            The exponent for the exponential decay filter. If None,
            it is determined automatically using `find_expocoef`.
        tstep : None or float
            The timeinterval between measurements. If None, it is
            determined automatically from the Time variable.
        noise_despiker : bool
            Whether or not to apply the standard deviation spike filter.
        win : int
            The rolling window over which the spike filter calculates
            the trace statistics.
        nlim : float
            The number of standard deviations above the rolling mean
            that data are excluded.
        exponentplot : bool
            Whether or not to show a plot of the automatically determined
            exponential decay exponent.
        maxiter : int
            The max number of times that the fitler is applied.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'rawdata'. Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.

        Returns
        -------
        None
        """
        if focus_stage != self.focus_stage:
            self.set_focus(focus_stage)

        if expdecay_despiker and exponent is None:
            if not hasattr(self, 'expdecay_coef'):
                self.find_expcoef(plot=exponentplot,
                                  autorange_kwargs=autorange_kwargs)
            exponent = self.expdecay_coef
            time.sleep(0.1)

        with self.pbar.set(total=len(self.data), desc='Despiking') as prog:
            for d in self.data.values():
                d.despike(expdecay_despiker, exponent,
                        noise_despiker, win, nlim, maxiter)
                prog.update()

        self.stages_complete.update(['despiked'])
        self.focus_stage = 'despiked'
        return

def despike ( self , expdecay_despiker = False , exponent = None , noise_despiker = True , win = 3 , nlim = 12. , exponentplot = False , maxiter = 4 , autorange_kwargs = { } , focus_stage = 'rawdata' ) : if focus_stage != self . focus_stage : self . set_focus ( focus_stage ) if expdecay_despiker and exponent is None : if not hasattr ( self , 'expdecay_coef' ) : self . find_expcoef ( plot = exponentplot , autorange_kwargs = autorange_kwargs ) exponent = self . expdecay_coef time . sleep ( 0.1 ) with self . pbar . set ( total = len ( self . data ) , desc = 'Despiking' ) as prog : for d in self . data . values ( ) : d . despike ( expdecay_despiker , exponent , noise_despiker , win , nlim , maxiter ) prog . update ( ) self . stages_complete . update ( [ 'despiked' ] ) self . focus_stage = 'despiked' return

Despikes data with exponential decay and noise filters .


def bkg_calc_weightedmean(self, analytes=None, weight_fwhm=None,
                              n_min=20, n_max=None, cstep=None,
                              bkg_filter=False, f_win=7, f_n_lim=3, focus_stage='despiked'):
        """
        Background calculation using a gaussian weighted mean.

        Parameters
        ----------
        analytes : str or iterable
            Which analyte or analytes to calculate.
        weight_fwhm : float
            The full-width-at-half-maximum of the gaussian used
            to calculate the weighted average.
        n_min : int
            Background regions with fewer than n_min points
            will not be included in the fit.
        cstep : float or None
            The interval between calculated background points.
        filter : bool
            If true, apply a rolling filter to the isolated background regions
            to exclude regions with anomalously high values. If True, two parameters
            alter the filter's behaviour:
        f_win : int
            The size of the rolling window
        f_n_lim : float
            The number of standard deviations above the rolling mean
            to set the threshold.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'despiked' if present, or 'rawdata' if not.
            Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.
        """
        if analytes is None:
            analytes = self.analytes
            self.bkg = Bunch()
        elif isinstance(analytes, str):
            analytes = [analytes]

        if weight_fwhm is None:
            weight_fwhm = 600  # 10 minute default window

        self.get_background(n_min=n_min, n_max=n_max,
                            bkg_filter=bkg_filter,
                            f_win=f_win, f_n_lim=f_n_lim, focus_stage=focus_stage)

        # Gaussian - weighted average
        if 'calc' not in self.bkg.keys():
            # create time points to calculate background
            if cstep is None:
                cstep = weight_fwhm / 20
            elif cstep > weight_fwhm:
                warnings.warn("\ncstep should be less than weight_fwhm. Your backgrounds\n" +
                              "might not behave as expected.\n")
            bkg_t = np.linspace(0,
                                self.max_time,
                                self.max_time // cstep)
            self.bkg['calc'] = Bunch()
            self.bkg['calc']['uTime'] = bkg_t

        # TODO : calculation then dict assignment is clumsy...
        mean, std, stderr = gauss_weighted_stats(self.bkg['raw'].uTime,
                                                 self.bkg['raw'].loc[:, analytes].values,
                                                 self.bkg['calc']['uTime'],
                                                 fwhm=weight_fwhm)

        for i, a in enumerate(analytes):
            self.bkg['calc'][a] = {'mean': mean[i],
                                    'std': std[i],
                                    'stderr': stderr[i]}

Background calculation using a gaussian weighted mean .


def bkg_calc_interp1d(self, analytes=None, kind=1, n_min=10, n_max=None, cstep=None,
                          bkg_filter=False, f_win=7, f_n_lim=3, focus_stage='despiked'):
        """
        Background calculation using a 1D interpolation.

        scipy.interpolate.interp1D is used for interpolation.

        Parameters
        ----------
        analytes : str or iterable
            Which analyte or analytes to calculate.
        kind : str or int
            Integer specifying the order of the spline interpolation
            used, or string specifying a type of interpolation.
            Passed to `scipy.interpolate.interp1D`
        n_min : int
            Background regions with fewer than n_min points
            will not be included in the fit.
        cstep : float or None
            The interval between calculated background points.
        filter : bool
            If true, apply a rolling filter to the isolated background regions
            to exclude regions with anomalously high values. If True, two parameters
            alter the filter's behaviour:
        f_win : int
            The size of the rolling window
        f_n_lim : float
            The number of standard deviations above the rolling mean
            to set the threshold.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'despiked' if present, or 'rawdata' if not.
            Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.
        """
        if analytes is None:
            analytes = self.analytes
            self.bkg = Bunch()
        elif isinstance(analytes, str):
            analytes = [analytes]

        self.get_background(n_min=n_min, n_max=n_max,
                            bkg_filter=bkg_filter,
                            f_win=f_win, f_n_lim=f_n_lim, focus_stage=focus_stage)

        def pad(a, lo=None, hi=None):
            if lo is None:
                lo = [a[0]]
            if hi is None:
                hi = [a[-1]]
            return np.concatenate((lo, a, hi))

        if 'calc' not in self.bkg.keys():
            # create time points to calculate background
            # if cstep is None:
                # cstep = self.bkg['raw']['uTime'].ptp() / 100
            # bkg_t = np.arange(self.bkg['summary']['uTime']['mean'].min(),
            #                   self.bkg['summary']['uTime']['mean'].max(),
            #                   cstep)

            bkg_t = pad(self.bkg['summary'].loc[:, ('uTime', 'mean')], [0], [self.max_time])

            self.bkg['calc'] = Bunch()
            self.bkg['calc']['uTime'] = bkg_t

        d = self.bkg['summary']
        with self.pbar.set(total=len(analytes), desc='Calculating Analyte Backgrounds') as prog:
            for a in analytes:
                self.bkg['calc'][a] = {'mean': pad(d.loc[:, (a, 'mean')].values),
                                    'std': pad(d.loc[:, (a, 'std')].values),
                                    'stderr': pad(d.loc[:, (a, 'stderr')].values)}
                prog.update()

        self.bkg['calc']

        return

Background calculation using a 1D interpolation .


def bkg_subtract(self, analytes=None, errtype='stderr', focus_stage='despiked'):
        """
        Subtract calculated background from data.

        Must run bkg_calc first!

        Parameters
        ----------
        analytes : str or iterable
            Which analyte(s) to subtract.
        errtype : str
            Which type of error to propagate. default is 'stderr'.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'despiked' if present, or 'rawdata' if not.
            Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.
        """
        if analytes is None:
            analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        if focus_stage == 'despiked':
            if 'despiked' not in self.stages_complete:
                focus_stage = 'rawdata'

        # make uncertainty-aware background interpolators
        bkg_interps = {}
        for a in analytes:
            bkg_interps[a] = un_interp1d(x=self.bkg['calc']['uTime'],
                                         y=un.uarray(self.bkg['calc'][a]['mean'],
                                                     self.bkg['calc'][a][errtype]))
        self.bkg_interps = bkg_interps

        # apply background corrections
        with self.pbar.set(total=len(self.data), desc='Background Subtraction') as prog:
            for d in self.data.values():
                # [d.bkg_subtract(a, bkg_interps[a].new(d.uTime), None, focus_stage=focus_stage) for a in analytes]
                [d.bkg_subtract(a, bkg_interps[a].new(d.uTime), ~d.sig, focus_stage=focus_stage) for a in analytes]
                d.setfocus('bkgsub')

                prog.update()

        self.stages_complete.update(['bkgsub'])
        self.focus_stage = 'bkgsub'
        return

Subtract calculated background from data .


def ratio(self, internal_standard=None):
        """
        Calculates the ratio of all analytes to a single analyte.

        Parameters
        ----------
        internal_standard : str
            The name of the analyte to divide all other analytes
            by.

        Returns
        -------
        None
        """
        if 'bkgsub' not in self.stages_complete:
            raise RuntimeError('Cannot calculate ratios before background subtraction.')

        if internal_standard is not None:
            self.internal_standard = internal_standard
            self.minimal_analytes.update([internal_standard])

        with self.pbar.set(total=len(self.data), desc='Ratio Calculation') as prog:
            for s in self.data.values():
                s.ratio(internal_standard=self.internal_standard)
                prog.update()

        self.stages_complete.update(['ratios'])
        self.focus_stage = 'ratios'
        return

Calculates the ratio of all analytes to a single analyte .


def make_subset(self, samples=None, name=None):
        """
        Creates a subset of samples, which can be treated independently.

        Parameters
        ----------
        samples : str or array - like
            Name of sample, or list of sample names.
        name : (optional) str or number
            The name of the sample group. Defaults to n + 1, where n is
            the highest existing group number
        """
        # Check if a subset containing the same samples already exists.
        for k, v in self.subsets.items():
            if set(v) == set(samples) and k != 'not_in_set':
                return k

        if isinstance(samples, str):
            samples = [samples]

        not_exists = [s for s in samples if s not in self.subsets['All_Analyses']]
        if len(not_exists) > 0:
            raise ValueError(', '.join(not_exists) + ' not in the list of sample names.\nPlease check your sample names.\nNote: Sample names are stored in the .samples attribute of your analysis.')

        if name is None:
            name = max([-1] + [x for x in self.subsets.keys() if isinstance(x, int)]) + 1

        self._subset_names.append(name)

        if samples is not None:
            self.subsets[name] = samples
            for s in samples:
                try:
                    self.subsets['not_in_set'].remove(s)
                except ValueError:
                    pass

        self._has_subsets = True

        # for subset in np.unique(list(self.subsets.values())):
        #     self.subsets[subset] = sorted([k for k, v in self.subsets.items() if str(v) == subset])

        return name

Creates a subset of samples which can be treated independently .


def filter_gradient_threshold_percentile(self, analyte, percentiles, level='population', win=15, filt=False,
                                             samples=None, subset=None):
        """
        Calculate a gradient threshold filter to the data.

        Generates two filters above and below the threshold value for a
        given analyte.

        Parameters
        ----------
        analyte : str
            The analyte that the filter applies to.
        win : int
            The window over which to calculate the moving gradient
        percentiles : float or iterable of len=2
            The percentile values.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        samples : array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.
        subset : str or number
            The subset of samples (defined by make_subset) you want to apply
            the filter to.

        Returns
        -------
        None
        """
        params = locals()
        del(params['self'])

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.minimal_analytes.update([analyte])

        # Calculate gradients of all samples
        self.get_gradients(analytes=[analyte], win=win, filt=filt, subset=subset)
        grad = self.gradients[analyte][~np.isnan(self.gradients[analyte])]

        if isinstance(percentiles, (int, float)):
            percentiles = [percentiles]

        if level == 'population':
            # calculate filter limits
            lims = np.percentile(grad, percentiles)

        # Calculate filter for individual samples
        with self.pbar.set(total=len(samples), desc='Percentile Threshold Filter') as prog:
            for s in samples:
                d = self.data[s]
                setn = d.filt.maxset + 1
                g = calc_grads(d.Time, d.focus, [analyte], win)[analyte]

                if level == 'individual':
                    gt = nominal_values(g)
                    lims = np.percentile(gt[~np.isnan(gt)], percentiles)

                if len(lims) == 1:
                    above = g >= lims[0]
                    below = g < lims[0]

                    d.filt.add(analyte + '_{:.1f}-grd-pcnt_below'.format(percentiles[0]),
                            below,
                            'Gradients below {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)
                    d.filt.add(analyte + '_{:.1f}-grd-pcnt_above'.format(percentiles[0]),
                            above,
                            'Gradients above {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)

                elif len(lims) == 2:
                    inside = (g >= min(lims)) & (g <= max(lims))
                    outside = (g < min(lims)) | (g > max(lims))

                    lpc = '-'.join(['{:.1f}'.format(p) for p in percentiles])
                    d.filt.add(analyte + '_' + lpc + '-grd-pcnt_inside',
                            inside,
                            'Gradients between ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                    d.filt.add(analyte + '_' + lpc + '-grd-pcnt_outside',
                            outside,
                            'Gradients outside ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                prog.update()
        return

Calculate a gradient threshold filter to the data .


def fit_classifier(self, name, analytes, method, samples=None,
                       subset=None, filt=True, sort_by=0, **kwargs):
        """
        Create a clustering classifier based on all samples, or a subset.

        Parameters
        ----------
        name : str
            The name of the classifier.
        analytes : str or iterable
            Which analytes the clustering algorithm should consider.
        method : str
            Which clustering algorithm to use. Can be:

            'meanshift'
                The `sklearn.cluster.MeanShift` algorithm.
                Automatically determines number of clusters
                in data based on the `bandwidth` of expected
                variation.
            'kmeans'
                The `sklearn.cluster.KMeans` algorithm. Determines
                the characteristics of a known number of clusters
                within the data. Must provide `n_clusters` to specify
                the expected number of clusters.
        samples : iterable
            list of samples to consider. Overrides 'subset'.
        subset : str
            The subset of samples used to fit the classifier. Ignored if
            'samples' is specified.
        sort_by : int
            Which analyte the resulting clusters should be sorted
            by - defaults to 0, which is the first analyte.
        **kwargs :
            method-specific keyword parameters - see below.
        Meanshift Parameters
            bandwidth : str or float
                The bandwith (float) or bandwidth method ('scott' or 'silverman')
                used to estimate the data bandwidth.
            bin_seeding : bool
                Modifies the behaviour of the meanshift algorithm. Refer to
                sklearn.cluster.meanshift documentation.
        K - Means Parameters
            n_clusters : int
                The number of clusters expected in the data.

        Returns
        -------
        name : str
        """
        # isolate data
        if samples is not None:
            subset = self.make_subset(samples)

        self.get_focus(subset=subset, filt=filt)

        # create classifer
        c = classifier(analytes,
                       sort_by)
        # fit classifier
        c.fit(data=self.focus,
              method=method,
              **kwargs)

        self.classifiers[name] = c

        return name

def
fit_classifier
(
self
,
name
,
analytes
,
method
,
samples
=
,
subset
=
,
filt
=
True
,
sort_by
=
0
,
*
*
kwargs
)
:
# isolate data
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
self
.
get_focus
(
subset
=
subset
,
filt
=
filt
)
# create classifer
c
=
classifier
(
analytes
,
sort_by
)
# fit classifier
c
.
fit
(
data
=
self
.
focus
,
method
=
method
,
*
*
kwargs
)
self
.
classifiers
[
name
]
=
c
return
name

Create a clustering classifier based on all samples or a subset .


def apply_classifier(self, name, samples=None, subset=None):
        """
        Apply a clustering classifier based on all samples, or a subset.

        Parameters
        ----------
        name : str
            The name of the classifier to apply.
        subset : str
            The subset of samples to apply the classifier to.
        Returns
        -------
        name : str
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        c = self.classifiers[name]
        labs = c.classifier.ulabels_

        with self.pbar.set(total=len(samples), desc='Applying ' + name + ' classifier') as prog:
            for s in samples:
                d = self.data[s]
                try:
                    f = c.predict(d.focus)
                except ValueError:
                    # in case there's no data
                    f = np.array([-2] * len(d.Time))
                for l in labs:
                    ind = f == l
                    d.filt.add(name=name + '_{:.0f}'.format(l),
                            filt=ind,
                            info=name + ' ' + c.method + ' classifier',
                            params=(c.analytes, c.method))
                prog.update()

        return name

def
apply_classifier
(
self
,
name
,
samples
=
,
subset
=
)
:
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
c
=
self
.
classifiers
[
name
]
labs
=
c
.
classifier
.
ulabels_
with
self
.
pbar
.
set
(
total
=
len
(
samples
)
,
desc
=
'Applying '
+
name
+
' classifier'
)
as
prog
:
for
s
in
samples
:
d
=
self
.
data
[
s
]
try
:
f
=
c
.
predict
(
d
.
focus
)
except
ValueError
:
# in case there's no data
f
=
np
.
array
(
[
2
]
*
len
(
d
.
Time
)
)
for
l
in
labs
:
ind
=
f
==
l
d
.
filt
.
add
(
name
=
name
+
'_{:.0f}'
.
format
(
l
)
,
filt
=
ind
,
info
=
name
+
' '
+
c
.
method
+
' classifier'
,
params
=
(
c
.
analytes
,
c
.
method
)
)
prog
.
update
(
)
return
name

Apply a clustering classifier based on all samples or a subset .


def filter_correlation(self, x_analyte, y_analyte, window=None,
                           r_threshold=0.9, p_threshold=0.05, filt=True,
                           samples=None, subset=None):
        """
        Applies a correlation filter to the data.

        Calculates a rolling correlation between every `window` points of
        two analytes, and excludes data where their Pearson's R value is
        above `r_threshold` and statistically significant.

        Data will be excluded where their absolute R value is greater than
        `r_threshold` AND the p - value associated with the correlation is
        less than `p_threshold`. i.e. only correlations that are statistically
        significant are considered.

        Parameters
        ----------
        x_analyte, y_analyte : str
            The names of the x and y analytes to correlate.
        window : int, None
            The rolling window used when calculating the correlation.
        r_threshold : float
            The correlation index above which to exclude data.
            Note: the absolute pearson R value is considered, so
            negative correlations below -`r_threshold` will also
            be excluded.
        p_threshold : float
            The significant level below which data are excluded.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.

        Returns
        -------
        None
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.minimal_analytes.update([x_analyte, y_analyte])

        with self.pbar.set(total=len(samples), desc='Correlation Filter') as prog:
            for s in samples:
                self.data[s].filter_correlation(x_analyte, y_analyte,
                                                window=window,
                                                r_threshold=r_threshold,
                                                p_threshold=p_threshold,
                                                filt=filt)
                prog.update()

def
filter_correlation
(
self
,
x_analyte
,
y_analyte
,
window
=
,
r_threshold
=
0.9
,
p_threshold
=
0.05
,
filt
=
True
,
samples
=
,
subset
=
)
:
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
self
.
minimal_analytes
.
update
(
[
x_analyte
,
y_analyte
]
)
with
self
.
pbar
.
set
(
total
=
len
(
samples
)
,
desc
=
'Correlation Filter'
)
as
prog
:
for
s
in
samples
:
self
.
data
[
s
]
.
filter_correlation
(
x_analyte
,
y_analyte
,
window
=
window
,
r_threshold
=
r_threshold
,
p_threshold
=
p_threshold
,
filt
=
filt
)
prog
.
update
(
)

Applies a correlation filter to the data .


def filter_on(self, filt=None, analyte=None, samples=None, subset=None, show_status=False):
        """
        Turns data filters on for particular analytes and samples.

        Parameters
        ----------
        filt : optional, str or array_like
            Name, partial name or list of names of filters. Supports
            partial matching. i.e. if 'cluster' is specified, all
            filters with 'cluster' in the name are activated.
            Defaults to all filters.
        analyte : optional, str or array_like
            Name or list of names of analytes. Defaults to all analytes.
        samples : optional, array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.

        Returns
        -------
        None
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        for s in samples:
            try:
                self.data[s].filt.on(analyte, filt)
            except:
                warnings.warn("filt.on failure in sample " + s)

        if show_status:
            self.filter_status(subset=subset)
        return

def
filter_on
(
self
,
filt
=
,
analyte
=
,
samples
=
,
subset
=
,
show_status
=
False
)
:
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
for
s
in
samples
:
try
:
self
.
data
[
s
]
.
filt
.
on
(
analyte
,
filt
)
except
:
warnings
.
warn
(
"filt.on failure in sample "
+
s
)
if
show_status
:
self
.
filter_status
(
subset
=
subset
)
return

Turns data filters on for particular analytes and samples .


def filter_off(self, filt=None, analyte=None, samples=None, subset=None, show_status=False):
        """
        Turns data filters off for particular analytes and samples.

        Parameters
        ----------
        filt : optional, str or array_like
            Name, partial name or list of names of filters. Supports
            partial matching. i.e. if 'cluster' is specified, all
            filters with 'cluster' in the name are activated.
            Defaults to all filters.
        analyte : optional, str or array_like
            Name or list of names of analytes. Defaults to all analytes.
        samples : optional, array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.

        Returns
        -------
        None
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        for s in samples:
            try:
                self.data[s].filt.off(analyte, filt)
            except:
                warnings.warn("filt.off failure in sample " + s)

        if show_status:
            self.filter_status(subset=subset)
        return

def
filter_off
(
self
,
filt
=
,
analyte
=
,
samples
=
,
subset
=
,
show_status
=
False
)
:
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
for
s
in
samples
:
try
:
self
.
data
[
s
]
.
filt
.
off
(
analyte
,
filt
)
except
:
warnings
.
warn
(
"filt.off failure in sample "
+
s
)
if
show_status
:
self
.
filter_status
(
subset
=
subset
)
return

Turns data filters off for particular analytes and samples .


def filter_status(self, sample=None, subset=None, stds=False):
        """
        Prints the current status of filters for specified samples.

        Parameters
        ----------
        sample : str
            Which sample to print.
        subset : str
            Specify a subset
        stds : bool
            Whether or not to include standards.
        """
        s = ''
        if sample is None and subset is None:
            if not self._has_subsets:
                s += 'Subset: All Samples\n\n'
                s += self.data[self.subsets['All_Samples'][0]].filt.__repr__()
            else:
                for n in sorted(str(sn) for sn in self._subset_names):
                    if n in self.subsets:
                        pass
                    elif int(n) in self.subsets:
                        n = int(n)
                        pass
                    s += 'Subset: ' + str(n) + '\n'
                    s += 'Samples: ' + ', '.join(self.subsets[n]) + '\n\n'
                    s += self.data[self.subsets[n][0]].filt.__repr__()
                if len(self.subsets['not_in_set']) > 0:
                    s += '\nNot in Subset:\n'
                    s += 'Samples: ' + ', '.join(self.subsets['not_in_set']) + '\n\n'
                    s += self.data[self.subsets['not_in_set'][0]].filt.__repr__()
            print(s)
            return

        elif sample is not None:
            s += 'Sample: ' + sample + '\n'
            s += self.data[sample].filt.__repr__()
            print(s)
            return

        elif subset is not None:
            if isinstance(subset, (str, int, float)):
                subset = [subset]
            for n in subset:
                s += 'Subset: ' + str(n) + '\n'
                s += 'Samples: ' + ', '.join(self.subsets[n]) + '\n\n'
                s += self.data[self.subsets[n][0]].filt.__repr__()
            print(s)
            return

def
filter_status
(
self
,
sample
=
,
subset
=
,
stds
=
False
)
:
s
=
''
if
sample
is
and
subset
is
:
if
not
self
.
_has_subsets
:
s
+=
'Subset: All Samples\n\n'
s
+=
self
.
data
[
self
.
subsets
[
'All_Samples'
]
[
0
]
]
.
filt
.
__repr__
(
)
else
:
for
n
in
sorted
(
str
(
sn
)
for
sn
in
self
.
_subset_names
)
:
if
n
in
self
.
subsets
:
pass
elif
int
(
n
)
in
self
.
subsets
:
n
=
int
(
n
)
pass
s
+=
'Subset: '
+
str
(
n
)
+
'\n'
s
+=
'Samples: '
+
', '
.
join
(
self
.
subsets
[
n
]
)
+
'\n\n'
s
+=
self
.
data
[
self
.
subsets
[
n
]
[
0
]
]
.
filt
.
__repr__
(
)
if
len
(
self
.
subsets
[
'not_in_set'
]
)
>
0
:
s
+=
'\nNot in Subset:\n'
s
+=
'Samples: '
+
', '
.
join
(
self
.
subsets
[
'not_in_set'
]
)
+
'\n\n'
s
+=
self
.
data
[
self
.
subsets
[
'not_in_set'
]
[
0
]
]
.
filt
.
__repr__
(
)
print
(
s
)
return
elif
sample
is
not
:
s
+=
'Sample: '
+
sample
+
'\n'
s
+=
self
.
data
[
sample
]
.
filt
.
__repr__
(
)
print
(
s
)
return
elif
subset
is
not
:
if
isinstance
(
subset
,
(
str
,
int
,
float
)
)
:
subset
=
[
subset
]
for
n
in
subset
:
s
+=
'Subset: '
+
str
(
n
)
+
'\n'
s
+=
'Samples: '
+
', '
.
join
(
self
.
subsets
[
n
]
)
+
'\n\n'
s
+=
self
.
data
[
self
.
subsets
[
n
]
[
0
]
]
.
filt
.
__repr__
(
)
print
(
s
)
return

Prints the current status of filters for specified samples .


def filter_defragment(self, threshold, mode='include', filt=True, samples=None, subset=None):
        """
        Remove 'fragments' from the calculated filter

        Parameters
        ----------
        threshold : int
            Contiguous data regions that contain this number
            or fewer points are considered 'fragments'
        mode : str
            Specifies wither to 'include' or 'exclude' the identified
            fragments.
        filt : bool or filt string
            Which filter to apply the defragmenter to. Defaults to True
        samples : array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.
        subset : str or number
            The subset of samples (defined by make_subset) you want to apply
            the filter to.

        Returns
        -------
        None
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        for s in samples:
            f = self.data[s].filt.grab_filt(filt)
            self.data[s].filt.add(name='defrag_{:s}_{:.0f}'.format(mode, threshold),
                                  filt=filters.defrag(f, threshold, mode),
                                  info='Defrag {:s} filter with threshold {:.0f}'.format(mode, threshold),
                                  params=(threshold, mode, filt, samples, subset))

Remove fragments from the calculated filter


def filter_nremoved(self, filt=True, quiet=False):
        """
        Report how many data are removed by the active filters.
        """
        rminfo = {}
        for n in self.subsets['All_Samples']:
            s = self.data[n]
            rminfo[n] = s.filt_nremoved(filt)
        if not quiet:
            maxL = max([len(s) for s in rminfo.keys()])
            print('{string:{number}s}'.format(string='Sample ', number=maxL + 3) +
                  '{total:4s}'.format(total='tot') +
                  '{removed:4s}'.format(removed='flt') +
                  '{percent:4s}'.format(percent='%rm'))
            for k, (ntot, nfilt, pcrm) in rminfo.items():
                print('{string:{number}s}'.format(string=k, number=maxL + 3) +
                      '{total:4.0f}'.format(total=ntot) +
                      '{removed:4.0f}'.format(removed=nfilt) +
                      '{percent:4.0f}'.format(percent=pcrm))

        return rminfo

def
filter_nremoved
(
self
,
filt
=
True
,
quiet
=
False
)
:
rminfo
=
{
}
for
n
in
self
.
subsets
[
'All_Samples'
]
:
s
=
self
.
data
[
n
]
rminfo
[
n
]
=
s
.
filt_nremoved
(
filt
)
if
not
quiet
:
maxL
=
max
(
[
len
(
s
)
for
s
in
rminfo
.
keys
(
)
]
)
print
(
'{string:{number}s}'
.
format
(
string
=
'Sample '
,
number
=
maxL
+
3
)
+
'{total:4s}'
.
format
(
total
=
'tot'
)
+
'{removed:4s}'
.
format
(
removed
=
'flt'
)
+
'{percent:4s}'
.
format
(
percent
=
'%rm'
)
)
for
k
,
(
ntot
,
nfilt
,
pcrm
)
in
rminfo
.
items
(
)
:
print
(
'{string:{number}s}'
.
format
(
string
=
k
,
number
=
maxL
+
3
)
+
'{total:4.0f}'
.
format
(
total
=
ntot
)
+
'{removed:4.0f}'
.
format
(
removed
=
nfilt
)
+
'{percent:4.0f}'
.
format
(
percent
=
pcrm
)
)
return
rminfo

Report how many data are removed by the active filters .


def gradient_histogram(self, analytes=None, win=15, filt=False, bins=None, samples=None, subset=None, recalc=True, ncol=4):
        """
        Plot a histogram of the gradients in all samples.

        Parameters
        ----------
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        bins : None or array-like
            The bins to use in the histogram
        samples : str or list
            which samples to get
        subset : str or int
            which subset to get
        recalc : bool
            Whether to re-calculate the gradients, or use existing gradients.

        Returns
        -------
        fig, ax
        """
        if analytes is None:
            analytes = [a for a in self.analytes if self.internal_standard not in a]
        if not hasattr(self, 'gradients'):
            self.gradients = Bunch()

        ncol = int(ncol)
        n = len(analytes)
        nrow = plot.calc_nrow(n, ncol)

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.get_gradients(analytes=analytes, win=win, filt=filt, subset=subset, recalc=recalc)

        fig, axs = plt.subplots(nrow, ncol, figsize=[3. * ncol, 2.5 * nrow])

        if not isinstance(axs, np.ndarray):
            axs = [axs]

        i = 0
        for a, ax in zip(analytes, axs.flatten()):
            d = nominal_values(self.gradients[a])
            d = d[~np.isnan(d)]

            m, u = unitpicker(d, focus_stage=self.focus_stage, denominator=self.internal_standard)

            if bins is None:
                ibins = np.linspace(*np.percentile(d * m, [1, 99]), 50)
            else:
                ibins = bins

            ax.hist(d * m, bins=ibins, color=self.cmaps[a])
            ax.axvline(0, ls='dashed', lw=1, c=(0,0,0,0.7))

            ax.set_title(a, loc='left')
            if ax.is_first_col():
                ax.set_ylabel('N')
            ax.set_xlabel(u + '/s')

            i += 1

        if i < ncol * nrow:
            for ax in axs.flatten()[i:]:
                ax.set_visible(False)

        fig.tight_layout()

        return fig, axs

def gradient_histogram ( self , analytes = None , win = 15 , filt = False , bins = None , samples = None , subset = None , recalc = True , ncol = 4 ) : if analytes is None : analytes = [ a for a in self . analytes if self . internal_standard not in a ] if not hasattr ( self , 'gradients' ) : self . gradients = Bunch ( ) ncol = int ( ncol ) n = len ( analytes ) nrow = plot . calc_nrow ( n , ncol ) if samples is not None : subset = self . make_subset ( samples ) samples = self . _get_samples ( subset ) self . get_gradients ( analytes = analytes , win = win , filt = filt , subset = subset , recalc = recalc ) fig , axs = plt . subplots ( nrow , ncol , figsize = [ 3. * ncol , 2.5 * nrow ] ) if not isinstance ( axs , np . ndarray ) : axs = [ axs ] i = 0 for a , ax in zip ( analytes , axs . flatten ( ) ) : d = nominal_values ( self . gradients [ a ] ) d = d [ ~ np . isnan ( d ) ] m , u = unitpicker ( d , focus_stage = self . focus_stage , denominator = self . internal_standard ) if bins is None : ibins = np . linspace ( * np . percentile ( d * m , [ 1 , 99 ] ) , 50 ) else : ibins = bins ax . hist ( d * m , bins = ibins , color = self . cmaps [ a ] ) ax . axvline ( 0 , ls = 'dashed' , lw = 1 , c = ( 0 , 0 , 0 , 0.7 ) ) ax . set_title ( a , loc = 'left' ) if ax . is_first_col ( ) : ax . set_ylabel ( 'N' ) ax . set_xlabel ( u + '/s' ) i += 1 if i < ncol * nrow : for ax in axs . flatten ( ) [ i : ] : ax . set_visible ( False ) fig . tight_layout ( ) return fig , axs

Plot a histogram of the gradients in all samples .


def gradient_crossplot(self, analytes=None, win=15, lognorm=True,
                           bins=25, filt=False, samples=None,
                           subset=None, figsize=(12, 12), save=False,
                           colourful=True, mode='hist2d', recalc=True, **kwargs):
        """
        Plot analyte gradients against each other.

        Parameters
        ----------
        analytes : optional, array_like or str
            The analyte(s) to plot. Defaults to all analytes.
        lognorm : bool
            Whether or not to log normalise the colour scale
            of the 2D histogram.
        bins : int
            The number of bins in the 2D histogram.
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        figsize : tuple
            Figure size (width, height) in inches.
        save : bool or str
            If True, plot is saves as 'crossplot.png', if str plot is
            saves as str.
        colourful : bool
            Whether or not the plot should be colourful :).
        mode : str
            'hist2d' (default) or 'scatter'
        recalc : bool
            Whether to re-calculate the gradients, or use existing gradients.

        Returns
        -------
        (fig, axes)
        """

        if analytes is None:
            analytes = self.analytes
        if self.focus_stage in ['ratio', 'calibrated']:
            analytes = [a for a in analytes if self.internal_standard not in a]

        # sort analytes
        try:
            analytes = sorted(analytes, key=lambda x: float(re.findall('[0-9.-]+', x)[0]))
        except IndexError:
            analytes = sorted(analytes)

        samples = self._get_samples(subset)

        # calculate gradients
        self.get_gradients(analytes=analytes, win=win, filt=filt, subset=subset, recalc=recalc)

        # self.get_focus(filt=filt, samples=samples, subset=subset)
        # grads = calc_grads(self.focus.uTime, self.focus, analytes, win)

        fig, axes = plot.crossplot(dat=self.gradients, keys=analytes, lognorm=lognorm,
                                   bins=bins, figsize=figsize, colourful=colourful,
                                   focus_stage=self.focus_stage, cmap=self.cmaps,
                                   denominator=self.internal_standard, mode=mode)

        if save:
            fig.savefig(self.report_dir + '/g_crossplot.png', dpi=200)

        return fig, axes

Plot analyte gradients against each other .


def histograms(self, analytes=None, bins=25, logy=False,
                   filt=False, colourful=True):
        """
        Plot histograms of analytes.

        Parameters
        ----------
        analytes : optional, array_like or str
            The analyte(s) to plot. Defaults to all analytes.
        bins : int
            The number of bins in each histogram (default = 25)
        logy : bool
            If true, y axis is a log scale.
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        colourful : bool
            If True, histograms are colourful :)

        Returns
        -------
        (fig, axes)
        """
        if analytes is None:
            analytes = self.analytes
        if self.focus_stage in ['ratio', 'calibrated']:
            analytes = [a for a in analytes if self.internal_standard not in a]
        if colourful:
            cmap = self.cmaps
        else:
            cmap = None

        self.get_focus(filt=filt)
        fig, axes = plot.histograms(self.focus, keys=analytes,
                                    bins=bins, logy=logy, cmap=cmap)

        return fig, axes

def histograms ( self , analytes = None , bins = 25 , logy = False , filt = False , colourful = True ) : if analytes is None : analytes = self . analytes if self . focus_stage in [ 'ratio' , 'calibrated' ] : analytes = [ a for a in analytes if self . internal_standard not in a ] if colourful : cmap = self . cmaps else : cmap = None self . get_focus ( filt = filt ) fig , axes = plot . histograms ( self . focus , keys = analytes , bins = bins , logy = logy , cmap = cmap ) return fig , axes


def trace_plots(self, analytes=None, samples=None, ranges=False,
                    focus=None, outdir=None, filt=None, scale='log',
                    figsize=[10, 4], stats=False, stat='nanmean',
                    err='nanstd', subset='All_Analyses'):
        """
        Plot analytes as a function of time.

        Parameters
        ----------
        analytes : optional, array_like or str
            The analyte(s) to plot. Defaults to all analytes.
        samples: optional, array_like or str
            The sample(s) to plot. Defaults to all samples.
        ranges : bool
            Whether or not to show the signal/backgroudn regions
            identified by 'autorange'.
        focus : str
            The focus 'stage' of the analysis to plot. Can be
            'rawdata', 'despiked':, 'signal', 'background',
            'bkgsub', 'ratios' or 'calibrated'.
        outdir : str
            Path to a directory where you'd like the plots to be
            saved. Defaults to 'reports/[focus]' in your data directory.
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        scale : str
            If 'log', plots the data on a log scale.
        figsize : array_like
            Array of length 2 specifying figure [width, height] in
            inches.
        stats : bool
            Whether or not to overlay the mean and standard deviations
            for each trace.
        stat, err: str
            The names of the statistic and error components to plot.
            Deafaults to 'nanmean' and 'nanstd'.

        Returns
        -------
        None
        """
        if focus is None:
            focus = self.focus_stage
        if outdir is None:
            outdir = self.report_dir + '/' + focus
        if not os.path.isdir(outdir):
            os.mkdir(outdir)

        # if samples is not None:
        #     subset = self.make_subset(samples)

        if subset is not None:
            samples = self._get_samples(subset)
        elif samples is None:
            samples = self.subsets['All_Analyses']
        elif isinstance(samples, str):
            samples = [samples]

        with self.pbar.set(total=len(samples), desc='Drawing Plots') as prog:
            for s in samples:
                f, a = self.data[s].tplot(analytes=analytes, figsize=figsize,
                                        scale=scale, filt=filt,
                                        ranges=ranges, stats=stats,
                                        stat=stat, err=err, focus_stage=focus)
                # ax = fig.axes[0]
                # for l, u in s.sigrng:
                #     ax.axvspan(l, u, color='r', alpha=0.1)
                # for l, u in s.bkgrng:
                #     ax.axvspan(l, u, color='k', alpha=0.1)
                f.savefig(outdir + '/' + s + '_traces.pdf')
                # TODO: on older(?) computers raises
                # 'OSError: [Errno 24] Too many open files'
                plt.close(f)
                prog.update()
        return

#     subset = self.make_subset(samples)


def gradient_plots(self, analytes=None, win=15, samples=None, ranges=False,
                       focus=None, outdir=None,
                       figsize=[10, 4], subset='All_Analyses'):
        """
        Plot analyte gradients as a function of time.

        Parameters
        ----------
        analytes : optional, array_like or str
            The analyte(s) to plot. Defaults to all analytes.
        samples: optional, array_like or str
            The sample(s) to plot. Defaults to all samples.
        ranges : bool
            Whether or not to show the signal/backgroudn regions
            identified by 'autorange'.
        focus : str
            The focus 'stage' of the analysis to plot. Can be
            'rawdata', 'despiked':, 'signal', 'background',
            'bkgsub', 'ratios' or 'calibrated'.
        outdir : str
            Path to a directory where you'd like the plots to be
            saved. Defaults to 'reports/[focus]' in your data directory.
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        scale : str
            If 'log', plots the data on a log scale.
        figsize : array_like
            Array of length 2 specifying figure [width, height] in
            inches.
        stats : bool
            Whether or not to overlay the mean and standard deviations
            for each trace.
        stat, err: str
            The names of the statistic and error components to plot.
            Deafaults to 'nanmean' and 'nanstd'.

        Returns
        -------
        None
        """
        if focus is None:
            focus = self.focus_stage
        if outdir is None:
            outdir = self.report_dir + '/' + focus + '_gradient'
        if not os.path.isdir(outdir):
            os.mkdir(outdir)

        # if samples is not None:
        #     subset = self.make_subset(samples)

        if subset is not None:
            samples = self._get_samples(subset)
        elif samples is None:
            samples = self.subsets['All_Analyses']
        elif isinstance(samples, str):
            samples = [samples]

        with self.pbar.set(total=len(samples), desc='Drawing Plots') as prog:
            for s in samples:
                f, a = self.data[s].gplot(analytes=analytes, win=win, figsize=figsize,
                                        ranges=ranges, focus_stage=focus)
                # ax = fig.axes[0]
                # for l, u in s.sigrng:
                #     ax.axvspan(l, u, color='r', alpha=0.1)
                # for l, u in s.bkgrng:
                #     ax.axvspan(l, u, color='k', alpha=0.1)
                f.savefig(outdir + '/' + s + '_gradients.pdf')
                # TODO: on older(?) computers raises
                # 'OSError: [Errno 24] Too many open files'
                plt.close(f)
                prog.update()
        return

Plot analyte gradients as a function of time .


def filter_reports(self, analytes, filt_str='all', nbin=5, samples=None,
                       outdir=None, subset='All_Samples'):
        """
        Plot filter reports for all filters that contain ``filt_str``
        in the name.
        """
        if outdir is None:
            outdir = self.report_dir + '/filters/' + filt_str
            if not os.path.isdir(self.report_dir + '/filters'):
                os.mkdir(self.report_dir + '/filters')
        if not os.path.isdir(outdir):
            os.mkdir(outdir)

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        with self.pbar.set(total=len(samples), desc='Drawing Plots') as prog:
            for s in samples:
                _ = self.data[s].filter_report(filt=filt_str,
                                            analytes=analytes,
                                            savedir=outdir,
                                            nbin=nbin)
                prog.update()
            # plt.close(fig)
        return

def
filter_reports
(
self
,
analytes
,
filt_str
=
'all'
,
nbin
=
5
,
samples
=
,
outdir
=
,
subset
=
'All_Samples'
)
:
if
outdir
is
:
outdir
=
self
.
report_dir
+
'/filters/'
+
filt_str
if
not
os
.
path
.
isdir
(
self
.
report_dir
+
'/filters'
)
:
os
.
mkdir
(
self
.
report_dir
+
'/filters'
)
if
not
os
.
path
.
isdir
(
outdir
)
:
os
.
mkdir
(
outdir
)
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
with
self
.
pbar
.
set
(
total
=
len
(
samples
)
,
desc
=
'Drawing Plots'
)
as
prog
:
for
s
in
samples
:
_
=
self
.
data
[
s
]
.
filter_report
(
filt
=
filt_str
,
analytes
=
analytes
,
savedir
=
outdir
,
nbin
=
nbin
)
prog
.
update
(
)
# plt.close(fig)
return

Plot filter reports for all filters that contain filt_str in the name .


def sample_stats(self, analytes=None, filt=True,
                     stats=['mean', 'std'],
                     eachtrace=True, csf_dict={}):
        """
        Calculate sample statistics.

        Returns samples, analytes, and arrays of statistics
        of shape (samples, analytes). Statistics are calculated
        from the 'focus' data variable, so output depends on how
        the data have been processed.

        Included stat functions:

        * :func:`~latools.stat_fns.mean`: arithmetic mean
        * :func:`~latools.stat_fns.std`: arithmetic standard deviation
        * :func:`~latools.stat_fns.se`: arithmetic standard error
        * :func:`~latools.stat_fns.H15_mean`: Huber mean (outlier removal)
        * :func:`~latools.stat_fns.H15_std`: Huber standard deviation (outlier removal)
        * :func:`~latools.stat_fns.H15_se`: Huber standard error (outlier removal)

        Parameters
        ----------
        analytes : optional, array_like or str
            The analyte(s) to calculate statistics for. Defaults to
            all analytes.
        filt : str, dict or bool
            Either logical filter expression contained in a str,
            a dict of expressions specifying the filter string to
            use for each analyte or a boolean. Passed to `grab_filt`.
        stats : array_like
            take a single array_like input, and return a single statistic.
            list of functions or names (see above) or functions that
            Function should be able to cope with NaN values.
        eachtrace : bool
            Whether to calculate the statistics for each analysis
            spot individually, or to produce per - sample means.
            Default is True.

        Returns
        -------
        None
            Adds dict to analyse object containing samples, analytes and
            functions and data.
        """
        if analytes is None:
            analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        self.stats = Bunch()

        self.stats_calced = []
        stat_fns = Bunch()

        stat_dict = {'mean': np.nanmean,
                     'std': np.nanstd,
                     'nanmean': np.nanmean,
                     'nanstd': np.nanstd,
                     'se': stderr,
                     'H15_mean': H15_mean,
                     'H15_std': H15_std,
                     'H15_se': H15_se}

        for s in stats:
            if isinstance(s, str):
                if s in stat_dict.keys():
                    self.stats_calced.append(s)
                    stat_fns[s] = stat_dict[s]
                if s in csf_dict.keys():
                    self.stats_calced.append(s)
                    exec(csf_dict[s])
                    stat_fns[s] = eval(s)
            elif callable(s):
                self.stats_calced.append(s.__name__)
                stat_fns[s.__name__] = s
                if not hasattr(self, 'custom_stat_functions'):
                    self.custom_stat_functions = ''
                self.custom_stat_functions += inspect.getsource(s) + '\n\n\n\n'

        # calculate stats for each sample
        with self.pbar.set(total=len(self.samples), desc='Calculating Stats') as prog:
            for s in self.samples:
                if self.srm_identifier not in s:
                    self.data[s].sample_stats(analytes, filt=filt,
                                            stat_fns=stat_fns,
                                            eachtrace=eachtrace)

                    self.stats[s] = self.data[s].stats
                prog.update()

def
sample_stats
(
self
,
analytes
=
,
filt
=
True
,
stats
=
[
'mean'
,
'std'
]
,
eachtrace
=
True
,
csf_dict
=
{
}
)
:
if
analytes
is
:
analytes
=
self
.
analytes
elif
isinstance
(
analytes
,
str
)
:
analytes
=
[
analytes
]
self
.
stats
=
Bunch
(
)
self
.
stats_calced
=
[
]
stat_fns
=
Bunch
(
)
stat_dict
=
{
'mean'
:
np
.
nanmean
,
'std'
:
np
.
nanstd
,
'nanmean'
:
np
.
nanmean
,
'nanstd'
:
np
.
nanstd
,
'se'
:
stderr
,
'H15_mean'
:
H15_mean
,
'H15_std'
:
H15_std
,
'H15_se'
:
H15_se
}
for
s
in
stats
:
if
isinstance
(
s
,
str
)
:
if
s
in
stat_dict
.
keys
(
)
:
self
.
stats_calced
.
append
(
s
)
stat_fns
[
s
]
=
stat_dict
[
s
]
if
s
in
csf_dict
.
keys
(
)
:
self
.
stats_calced
.
append
(
s
)
exec
(
csf_dict
[
s
]
)
stat_fns
[
s
]
=
eval
(
s
)
elif
callable
(
s
)
:
self
.
stats_calced
.
append
(
s
.
__name__
)
stat_fns
[
s
.
__name__
]
=
s
if
not
hasattr
(
self
,
'custom_stat_functions'
)
:
self
.
custom_stat_functions
=
''
self
.
custom_stat_functions
+=
inspect
.
getsource
(
s
)
+
'\n\n\n\n'
# calculate stats for each sample
with
self
.
pbar
.
set
(
total
=
len
(
self
.
samples
)
,
desc
=
'Calculating Stats'
)
as
prog
:
for
s
in
self
.
samples
:
if
self
.
srm_identifier
not
in
s
:
self
.
data
[
s
]
.
sample_stats
(
analytes
,
filt
=
filt
,
stat_fns
=
stat_fns
,
eachtrace
=
eachtrace
)
self
.
stats
[
s
]
=
self
.
data
[
s
]
.
stats
prog
.
update
(
)


def getstats(self, save=True, filename=None, samples=None, subset=None, ablation_time=False):
        """
        Return pandas dataframe of all sample statistics.
        """
        slst = []

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        for s in self.stats_calced:
            for nm in [n for n in samples if self.srm_identifier
                       not in n]:
                if self.stats[nm][s].ndim == 2:
                    # make multi - index
                    reps = np.arange(self.stats[nm][s].shape[-1])
                    ss = np.array([s] * reps.size)
                    nms = np.array([nm] * reps.size)
                    # make sub - dataframe
                    stdf = pd.DataFrame(self.stats[nm][s].T,
                                        columns=self.stats[nm]['analytes'],
                                        index=[ss, nms, reps])
                    stdf.index.set_names(['statistic', 'sample', 'rep'],
                                         inplace=True)
                else:
                    stdf = pd.DataFrame(self.stats[nm][s],
                                        index=self.stats[nm]['analytes'],
                                        columns=[[s], [nm]]).T

                    stdf.index.set_names(['statistic', 'sample'],
                                         inplace=True)
                slst.append(stdf)
        out = pd.concat(slst)

        if ablation_time:
            ats = self.ablation_times(samples=samples, subset=subset)
            ats['statistic'] = 'nanmean'
            ats.set_index('statistic', append=True, inplace=True)
            ats = ats.reorder_levels(['statistic', 'sample', 'rep'])

            out = out.join(ats)

        out.drop(self.internal_standard, 1, inplace=True)

        if save:
            if filename is None:
                filename = 'stat_export.csv'
            out.to_csv(self.export_dir + '/' + filename)

        self.stats_df = out

        return out

def
getstats
(
self
,
save
=
True
,
filename
=
,
samples
=
,
subset
=
,
ablation_time
=
False
)
:
slst
=
[
]
if
samples
is
not
:
subset
=
self
.
make_subset
(
samples
)
samples
=
self
.
_get_samples
(
subset
)
for
s
in
self
.
stats_calced
:
for
nm
in
[
n
for
n
in
samples
if
self
.
srm_identifier
not
in
n
]
:
if
self
.
stats
[
nm
]
[
s
]
.
ndim
==
2
:
# make multi - index
reps
=
np
.
arange
(
self
.
stats
[
nm
]
[
s
]
.
shape
[
1
]
)
ss
=
np
.
array
(
[
s
]
*
reps
.
size
)
nms
=
np
.
array
(
[
nm
]
*
reps
.
size
)
# make sub - dataframe
stdf
=
pd
.
DataFrame
(
self
.
stats
[
nm
]
[
s
]
.
T
,
columns
=
self
.
stats
[
nm
]
[
'analytes'
]
,
index
=
[
ss
,
nms
,
reps
]
)
stdf
.
index
.
set_names
(
[
'statistic'
,
'sample'
,
'rep'
]
,
inplace
=
True
)
else
:
stdf
=
pd
.
DataFrame
(
self
.
stats
[
nm
]
[
s
]
,
index
=
self
.
stats
[
nm
]
[
'analytes'
]
,
columns
=
[
[
s
]
,
[
nm
]
]
)
.
T
stdf
.
index
.
set_names
(
[
'statistic'
,
'sample'
]
,
inplace
=
True
)
slst
.
append
(
stdf
)
out
=
pd
.
concat
(
slst
)
if
ablation_time
:
ats
=
self
.
ablation_times
(
samples
=
samples
,
subset
=
subset
)
ats
[
'statistic'
]
=
'nanmean'
ats
.
set_index
(
'statistic'
,
append
=
True
,
inplace
=
True
)
ats
=
ats
.
reorder_levels
(
[
'statistic'
,
'sample'
,
'rep'
]
)
out
=
out
.
join
(
ats
)
out
.
drop
(
self
.
internal_standard
,
1
,
inplace
=
True
)
if
save
:
if
filename
is
:
filename
=
'stat_export.csv'
out
.
to_csv
(
self
.
export_dir
+
'/'
+
filename
)
self
.
stats_df
=
out
return
out

Return pandas dataframe of all sample statistics .


def _minimal_export_traces(self, outdir=None, analytes=None,
                               samples=None, subset='All_Analyses'):
        """
        Used for exporting minimal dataset. DON'T USE.
        """
        if analytes is None:
            analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        focus_stage = 'rawdata'
        # ud = 'counts'

        if not os.path.isdir(outdir):
            os.mkdir(outdir)

        for s in samples:
            d = self.data[s].data[focus_stage]
            out = Bunch()

            for a in analytes:
                out[a] = d[a]

            out = pd.DataFrame(out, index=self.data[s].Time)
            out.index.name = 'Time'

            d = dateutil.parser.parse(self.data[s].meta['date'])
            header = ['# Minimal Reproduction Dataset Exported from LATOOLS on %s' %
                      (time.strftime('%Y:%m:%d %H:%M:%S')),
                      "# Analysis described in '../analysis.lalog'",
                      '# Run latools.reproduce to import analysis.',
                      '#',
                      '# Sample: %s' % (s),
                      '# Analysis Time: ' + d.strftime('%Y-%m-%d %H:%M:%S')]

            header = '\n'.join(header) + '\n'

            csv = out.to_csv()

            with open('%s/%s.csv' % (outdir, s), 'w') as f:
                f.write(header)
                f.write(csv)
        return

Used for exporting minimal dataset . DON T USE .