def _transit_model(times, t0, per, rp, a, inc, ecc, w, u, limb_dark,
                   exp_time_minutes=2, supersample_factor=7):
    '''This returns a BATMAN planetary transit model.

    Parameters
    ----------

    times : np.array
        The times at which the model will be evaluated.

    t0 : float
        The time of periastron for the transit.

    per : float
        The orbital period of the planet.

    rp : float
        The stellar radius of the planet's star (in Rsun).

    a : float
        The semi-major axis of the planet's orbit (in Rsun).

    inc : float
        The orbital inclination (in degrees).

    ecc : float
        The eccentricity of the orbit.

    w : float
        The longitude of periastron (in degrees).

    u : list of floats
        The limb darkening coefficients specific to the limb darkening model
        used.

    limb_dark : {"uniform", "linear", "quadratic", "square-root", "logarithmic", "exponential", "power2", "custom"}
        The type of limb darkening model to use. See the full list here:

        https://www.cfa.harvard.edu/~lkreidberg/batman/tutorial.html#limb-darkening-options

    exp_time_minutes : float
        The amount of time to 'smear' the transit LC points over to simulate a
        long exposure time.

    supersample_factor: int
        The number of supersampled time data points to average the lightcurve
        model over.

    Returns
    -------

    (params, batman_model) : tuple
        The returned tuple contains the params list and the generated
        `batman.TransitModel` object.

    '''
    params = batman.TransitParams()  # object to store transit parameters
    params.t0 = t0                   # time of periastron
    params.per = per                 # orbital period
    params.rp = rp                   # planet radius (in stellar radii)
    params.a = a                     # semi-major axis (in stellar radii)
    params.inc = inc                 # orbital inclination (in degrees)
    params.ecc = ecc                 # the eccentricity of the orbit
    params.w = w                     # longitude of periastron (in degrees)
    params.u = u                     # limb darkening coefficient list
    params.limb_dark = limb_dark     # limb darkening model to use

    t = times
    m = batman.TransitModel(params, t, exp_time=exp_time_minutes/60./24.,
                            supersample_factor=supersample_factor)

    return params, m

def
_transit_model
(
times
,
t0
,
per
,
rp
,
a
,
inc
,
ecc
,
w
,
u
,
limb_dark
,
exp_time_minutes
=
2
,
supersample_factor
=
7
)
:
params
=
batman
.
TransitParams
(
)
# object to store transit parameters
params
.
t0
=
t0
# time of periastron
params
.
per
=
per
# orbital period
params
.
rp
=
rp
# planet radius (in stellar radii)
params
.
a
=
a
# semi-major axis (in stellar radii)
params
.
inc
=
inc
# orbital inclination (in degrees)
params
.
ecc
=
ecc
# the eccentricity of the orbit
params
.
w
=
w
# longitude of periastron (in degrees)
params
.
u
=
u
# limb darkening coefficient list
params
.
limb_dark
=
limb_dark
# limb darkening model to use
t
=
times
m
=
batman
.
TransitModel
(
params
,
t
,
exp_time
=
exp_time_minutes
/
60.
/
24.
,
supersample_factor
=
supersample_factor
)
return
params
,
m

This returns a BATMAN planetary transit model .


def _log_prior_transit(theta, priorbounds):
    '''
    Assume priors on all parameters have uniform probability.
    '''
    # priorbounds contains the input priors, and because of how we previously
    # sorted theta, its sorted keys tell us which parts of theta correspond to
    # which physical quantities.

    allowed = True
    for ix, key in enumerate(np.sort(list(priorbounds.keys()))):
        if priorbounds[key][0] < theta[ix] < priorbounds[key][1]:
            allowed = True and allowed
        else:
            allowed = False

    if allowed:
        return 0.

    return -np.inf

Assume priors on all parameters have uniform probability .


def list_trilegal_filtersystems():
    '''
    This just lists all the filter systems available for TRILEGAL.

    '''

    print('%-40s %s' % ('FILTER SYSTEM NAME','DESCRIPTION'))
    print('%-40s %s' % ('------------------','-----------'))
    for key in sorted(TRILEGAL_FILTER_SYSTEMS.keys()):
        print('%-40s %s' % (key, TRILEGAL_FILTER_SYSTEMS[key]['desc']))

def
list_trilegal_filtersystems
(
)
:
print
(
'%-40s %s'
%
(
'FILTER SYSTEM NAME'
,
'DESCRIPTION'
)
)
print
(
'%-40s %s'
%
(
'------------------'
,
'-----------'
)
)
for
key
in
sorted
(
TRILEGAL_FILTER_SYSTEMS
.
keys
(
)
)
:
print
(
'%-40s %s'
%
(
key
,
TRILEGAL_FILTER_SYSTEMS
[
key
]
[
'desc'
]
)
)

This just lists all the filter systems available for TRILEGAL .


def query_radecl(ra,
                 decl,
                 filtersystem='sloan_2mass',
                 field_deg2=1.0,
                 usebinaries=True,
                 extinction_sigma=0.1,
                 magnitude_limit=26.0,
                 maglim_filtercol=4,
                 trilegal_version=1.6,
                 extraparams=None,
                 forcefetch=False,
                 cachedir='~/.astrobase/trilegal-cache',
                 verbose=True,
                 timeout=60.0,
                 refresh=150.0,
                 maxtimeout=700.0):
    '''This runs the TRILEGAL query for decimal equatorial coordinates.

    Parameters
    ----------

    ra,decl : float
        These are the center equatorial coordinates in decimal degrees

    filtersystem : str
        This is a key in the TRILEGAL_FILTER_SYSTEMS dict. Use the function
        :py:func:`astrobase.services.trilegal.list_trilegal_filtersystems` to
        see a nicely formatted table with the key and description for each of
        these.

    field_deg2 : float
        The area of the simulated field in square degrees. This is in the
        Galactic coordinate system.

    usebinaries : bool
        If this is True, binaries will be present in the model results.

    extinction_sigma : float
        This is the applied std dev around the `Av_extinction` value for the
        galactic coordinates requested.

    magnitude_limit : float
        This is the limiting magnitude of the simulation in the
        `maglim_filtercol` band index of the filter system chosen.

    maglim_filtercol : int
        The index in the filter system list of the magnitude limiting band.

    trilegal_version : float
        This is the the version of the TRILEGAL form to use. This can usually be
        left as-is.

    extraparams : dict or None
        This is a dict that can be used to override parameters of the model
        other than the basic ones used for input to this function. All
        parameters are listed in `TRILEGAL_DEFAULT_PARAMS` above. See:

        http://stev.oapd.inaf.it/cgi-bin/trilegal

        for explanations of these parameters.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    Returns
    -------

    dict
        This returns a dict of the form::

            {'params':the input param dict used,
             'extraparams':any extra params used,
             'provenance':'cached' or 'new download',
             'tablefile':the path on disk to the downloaded model text file}

    '''

    # convert the ra/decl to gl, gb
    radecl = SkyCoord(ra=ra*u.degree, dec=decl*u.degree)

    gl = radecl.galactic.l.degree
    gb = radecl.galactic.b.degree

    return query_galcoords(gl,
                           gb,
                           filtersystem=filtersystem,
                           field_deg2=field_deg2,
                           usebinaries=usebinaries,
                           extinction_sigma=extinction_sigma,
                           magnitude_limit=magnitude_limit,
                           maglim_filtercol=maglim_filtercol,
                           trilegal_version=trilegal_version,
                           extraparams=extraparams,
                           forcefetch=forcefetch,
                           cachedir=cachedir,
                           verbose=verbose,
                           timeout=timeout,
                           refresh=refresh,
                           maxtimeout=maxtimeout)

def
query_radecl
(
ra
,
decl
,
filtersystem
=
'sloan_2mass'
,
field_deg2
=
1.0
,
usebinaries
=
True
,
extinction_sigma
=
0.1
,
magnitude_limit
=
26.0
,
maglim_filtercol
=
4
,
trilegal_version
=
1.6
,
extraparams
=
,
forcefetch
=
False
,
cachedir
=
'~/.astrobase/trilegal-cache'
,
verbose
=
True
,
timeout
=
60.0
,
refresh
=
150.0
,
maxtimeout
=
700.0
)
:
# convert the ra/decl to gl, gb
radecl
=
SkyCoord
(
ra
=
ra
*
u
.
degree
,
dec
=
decl
*
u
.
degree
)
gl
=
radecl
.
galactic
.
l
.
degree
gb
=
radecl
.
galactic
.
b
.
degree
return
query_galcoords
(
gl
,
gb
,
filtersystem
=
filtersystem
,
field_deg2
=
field_deg2
,
usebinaries
=
usebinaries
,
extinction_sigma
=
extinction_sigma
,
magnitude_limit
=
magnitude_limit
,
maglim_filtercol
=
maglim_filtercol
,
trilegal_version
=
trilegal_version
,
extraparams
=
extraparams
,
forcefetch
=
forcefetch
,
cachedir
=
cachedir
,
verbose
=
verbose
,
timeout
=
timeout
,
refresh
=
refresh
,
maxtimeout
=
maxtimeout
)

This runs the TRILEGAL query for decimal equatorial coordinates .


def read_model_table(modelfile):
    '''
    This reads a downloaded TRILEGAL model file.

    Parameters
    ----------

    modelfile : str
        Path to the downloaded model file to read.

    Returns
    -------

    np.recarray
        Returns the model table as a Numpy record array.

    '''

    infd = gzip.open(modelfile)
    model = np.genfromtxt(infd,names=True)
    infd.close()

    return model

def read_model_table ( modelfile ) : infd = gzip . open ( modelfile ) model = np . genfromtxt ( infd , names = True ) infd . close ( ) return model

This reads a downloaded TRILEGAL model file .


def _time_independent_equals(a, b):
    '''
    This compares two values in constant time.

    Taken from tornado:

    https://github.com/tornadoweb/tornado/blob/
    d4eb8eb4eb5cc9a6677e9116ef84ded8efba8859/tornado/web.py#L3060

    '''
    if len(a) != len(b):
        return False
    result = 0
    if isinstance(a[0], int):  # python3 byte strings
        for x, y in zip(a, b):
            result |= x ^ y
    else:  # python2
        for x, y in zip(a, b):
            result |= ord(x) ^ ord(y)
    return result == 0

def
_time_independent_equals
(
a
,
b
)
:
if
len
(
a
)
!=
len
(
b
)
:
return
False
result
=
0
if
isinstance
(
a
[
0
]
,
int
)
:
# python3 byte strings
for
x
,
y
in
zip
(
a
,
b
)
:
result
|=
x
^
y
else
:
# python2
for
x
,
y
in
zip
(
a
,
b
)
:
result
|=
ord
(
x
)
^
ord
(
y
)
return
result
==
0

This compares two values in constant time .


def default(self, obj):
        '''Overrides the default serializer for `JSONEncoder`.

        This can serialize the following objects in addition to what
        `JSONEncoder` can already do.

        - `np.array`
        - `bytes`
        - `complex`
        - `np.float64` and other `np.dtype` objects

        Parameters
        ----------

        obj : object
            A Python object to serialize to JSON.

        Returns
        -------

        str
            A JSON encoded representation of the input object.

        '''

        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, bytes):
            return obj.decode()
        elif isinstance(obj, complex):
            return (obj.real, obj.imag)
        elif (isinstance(obj, (float, np.float64, np.float_)) and
              not np.isfinite(obj)):
            return None
        elif isinstance(obj, (np.int8, np.int16, np.int32, np.int64)):
            return int(obj)
        else:
            return json.JSONEncoder.default(self, obj)

def default ( self , obj ) : if isinstance ( obj , np . ndarray ) : return obj . tolist ( ) elif isinstance ( obj , bytes ) : return obj . decode ( ) elif isinstance ( obj , complex ) : return ( obj . real , obj . imag ) elif ( isinstance ( obj , ( float , np . float64 , np . float_ ) ) and not np . isfinite ( obj ) ) : return None elif isinstance ( obj , ( np . int8 , np . int16 , np . int32 , np . int64 ) ) : return int ( obj ) else : return json . JSONEncoder . default ( self , obj )

Overrides the default serializer for JSONEncoder .


def initialize(self, currentdir, assetpath, cplist,
                   cplistfile, executor, readonly, baseurl):
        '''
        handles initial setup.

        '''

        self.currentdir = currentdir
        self.assetpath = assetpath
        self.currentproject = cplist
        self.cplistfile = cplistfile
        self.executor = executor
        self.readonly = readonly
        self.baseurl = baseurl

def initialize ( self , currentdir , assetpath , cplist , cplistfile , executor , readonly , baseurl ) : self . currentdir = currentdir self . assetpath = assetpath self . currentproject = cplist self . cplistfile = cplistfile self . executor = executor self . readonly = readonly self . baseurl = baseurl


def get(self):
        '''This handles GET requests to the index page.

        TODO: provide the correct baseurl from the checkplotserver options dict,
        so the frontend JS can just read that off immediately.

        '''

        # generate the project's list of checkplots
        project_checkplots = self.currentproject['checkplots']
        project_checkplotbasenames = [os.path.basename(x)
                                      for x in project_checkplots]
        project_checkplotindices = range(len(project_checkplots))

        # get the sortkey and order
        project_cpsortkey = self.currentproject['sortkey']
        if self.currentproject['sortorder'] == 'asc':
            project_cpsortorder = 'ascending'
        elif self.currentproject['sortorder'] == 'desc':
            project_cpsortorder = 'descending'

        # get the filterkey and condition
        project_cpfilterstatements = self.currentproject['filterstatements']

        self.render('cpindex.html',
                    project_checkplots=project_checkplots,
                    project_cpsortorder=project_cpsortorder,
                    project_cpsortkey=project_cpsortkey,
                    project_cpfilterstatements=project_cpfilterstatements,
                    project_checkplotbasenames=project_checkplotbasenames,
                    project_checkplotindices=project_checkplotindices,
                    project_checkplotfile=self.cplistfile,
                    readonly=self.readonly,
                    baseurl=self.baseurl)

This handles GET requests to the index page .


def get(self):
        '''
        This handles GET requests for the current checkplot-list.json file.

        Used with AJAX from frontend.

        '''

        # add the reviewed key to the current dict if it doesn't exist
        # this will hold all the reviewed objects for the frontend
        if 'reviewed' not in self.currentproject:
            self.currentproject['reviewed'] = {}

        # just returns the current project as JSON
        self.write(self.currentproject)

This handles GET requests for the current checkplot - list . json file .


def initialize(self, executor, secret):
        '''
        This handles initial setup of the `RequestHandler`.

        '''

        self.executor = executor
        self.secret = secret

def initialize ( self , executor , secret ) : self . executor = executor self . secret = secret

This handles initial setup of the RequestHandler .


def smooth_magseries_gaussfilt(mags, windowsize, windowfwhm=7):
    '''This smooths the magseries with a Gaussian kernel.

    Parameters
    ----------

    mags : np.array
        The input mags/flux time-series to smooth.

    windowsize : int
        This is a odd integer containing the smoothing window size.

    windowfwhm : int
        This is an odd integer containing the FWHM of the applied Gaussian
        window function.

    Returns
    -------

    np.array
        The smoothed mag/flux time-series array.

    '''

    convkernel = Gaussian1DKernel(windowfwhm, x_size=windowsize)
    smoothed = convolve(mags, convkernel, boundary='extend')
    return smoothed

def smooth_magseries_gaussfilt ( mags , windowsize , windowfwhm = 7 ) : convkernel = Gaussian1DKernel ( windowfwhm , x_size = windowsize ) smoothed = convolve ( mags , convkernel , boundary = 'extend' ) return smoothed

This smooths the magseries with a Gaussian kernel .


def smooth_magseries_savgol(mags, windowsize, polyorder=2):
    '''This smooths the magseries with a Savitsky-Golay filter.

    Parameters
    ----------

    mags : np.array
        The input mags/flux time-series to smooth.

    windowsize : int
        This is a odd integer containing the smoothing window size.

    polyorder : int
        This is an integer containing the polynomial degree order to use when
        generating the Savitsky-Golay filter.

    Returns
    -------

    np.array
        The smoothed mag/flux time-series array.

    '''

    smoothed = savgol_filter(mags, windowsize, polyorder)
    return smoothed

def smooth_magseries_savgol ( mags , windowsize , polyorder = 2 ) : smoothed = savgol_filter ( mags , windowsize , polyorder ) return smoothed

This smooths the magseries with a Savitsky - Golay filter .


def _old_epd_diffmags(coeff, fsv, fdv, fkv, xcc, ycc, bgv, bge, mag):
    '''
    This calculates the difference in mags after EPD coefficients are
    calculated.

    final EPD mags = median(magseries) + epd_diffmags()

    '''

    return -(coeff[0]*fsv**2. +
             coeff[1]*fsv +
             coeff[2]*fdv**2. +
             coeff[3]*fdv +
             coeff[4]*fkv**2. +
             coeff[5]*fkv +
             coeff[6] +
             coeff[7]*fsv*fdv +
             coeff[8]*fsv*fkv +
             coeff[9]*fdv*fkv +
             coeff[10]*np.sin(2*np.pi*xcc) +
             coeff[11]*np.cos(2*np.pi*xcc) +
             coeff[12]*np.sin(2*np.pi*ycc) +
             coeff[13]*np.cos(2*np.pi*ycc) +
             coeff[14]*np.sin(4*np.pi*xcc) +
             coeff[15]*np.cos(4*np.pi*xcc) +
             coeff[16]*np.sin(4*np.pi*ycc) +
             coeff[17]*np.cos(4*np.pi*ycc) +
             coeff[18]*bgv +
             coeff[19]*bge -
             mag)

def _old_epd_diffmags ( coeff , fsv , fdv , fkv , xcc , ycc , bgv , bge , mag ) : return - ( coeff [ 0 ] * fsv ** 2. + coeff [ 1 ] * fsv + coeff [ 2 ] * fdv ** 2. + coeff [ 3 ] * fdv + coeff [ 4 ] * fkv ** 2. + coeff [ 5 ] * fkv + coeff [ 6 ] + coeff [ 7 ] * fsv * fdv + coeff [ 8 ] * fsv * fkv + coeff [ 9 ] * fdv * fkv + coeff [ 10 ] * np . sin ( 2 * np . pi * xcc ) + coeff [ 11 ] * np . cos ( 2 * np . pi * xcc ) + coeff [ 12 ] * np . sin ( 2 * np . pi * ycc ) + coeff [ 13 ] * np . cos ( 2 * np . pi * ycc ) + coeff [ 14 ] * np . sin ( 4 * np . pi * xcc ) + coeff [ 15 ] * np . cos ( 4 * np . pi * xcc ) + coeff [ 16 ] * np . sin ( 4 * np . pi * ycc ) + coeff [ 17 ] * np . cos ( 4 * np . pi * ycc ) + coeff [ 18 ] * bgv + coeff [ 19 ] * bge - mag )

This calculates the difference in mags after EPD coefficients are calculated .


def _old_epd_magseries(times, mags, errs,
                       fsv, fdv, fkv, xcc, ycc, bgv, bge,
                       epdsmooth_windowsize=21,
                       epdsmooth_sigclip=3.0,
                       epdsmooth_func=smooth_magseries_signal_medfilt,
                       epdsmooth_extraparams=None):
    '''
    Detrends a magnitude series given in mag using accompanying values of S in
    fsv, D in fdv, K in fkv, x coords in xcc, y coords in ycc, background in
    bgv, and background error in bge. smooth is used to set a smoothing
    parameter for the fit function. Does EPD voodoo.

    '''

    # find all the finite values of the magsnitude
    finiteind = np.isfinite(mags)

    # calculate median and stdev
    mags_median = np.median(mags[finiteind])
    mags_stdev = np.nanstd(mags)

    # if we're supposed to sigma clip, do so
    if epdsmooth_sigclip:
        excludeind = abs(mags - mags_median) < epdsmooth_sigclip*mags_stdev
        finalind = finiteind & excludeind
    else:
        finalind = finiteind

    final_mags = mags[finalind]
    final_len = len(final_mags)

    # smooth the signal
    if isinstance(epdsmooth_extraparams, dict):
        smoothedmags = epdsmooth_func(final_mags,
                                      epdsmooth_windowsize,
                                      **epdsmooth_extraparams)
    else:
        smoothedmags = epdsmooth_func(final_mags, epdsmooth_windowsize)

    # make the linear equation matrix
    epdmatrix = np.c_[fsv[finalind]**2.0,
                      fsv[finalind],
                      fdv[finalind]**2.0,
                      fdv[finalind],
                      fkv[finalind]**2.0,
                      fkv[finalind],
                      np.ones(final_len),
                      fsv[finalind]*fdv[finalind],
                      fsv[finalind]*fkv[finalind],
                      fdv[finalind]*fkv[finalind],
                      np.sin(2*np.pi*xcc[finalind]),
                      np.cos(2*np.pi*xcc[finalind]),
                      np.sin(2*np.pi*ycc[finalind]),
                      np.cos(2*np.pi*ycc[finalind]),
                      np.sin(4*np.pi*xcc[finalind]),
                      np.cos(4*np.pi*xcc[finalind]),
                      np.sin(4*np.pi*ycc[finalind]),
                      np.cos(4*np.pi*ycc[finalind]),
                      bgv[finalind],
                      bge[finalind]]

    # solve the matrix equation [epdmatrix] . [x] = [smoothedmags]
    # return the EPD differential magss if the solution succeeds
    try:

        coeffs, residuals, rank, singulars = lstsq(epdmatrix, smoothedmags,
                                                   rcond=None)

        if DEBUG:
            print('coeffs = %s, residuals = %s' % (coeffs, residuals))

        retdict = {'times':times,
                   'mags':(mags_median +
                           _old_epd_diffmags(coeffs, fsv, fdv,
                                             fkv, xcc, ycc, bgv, bge, mags)),
                   'errs':errs,
                   'fitcoeffs':coeffs,
                   'residuals':residuals}

        return retdict

    # if the solution fails, return nothing
    except Exception as e:

        LOGEXCEPTION('EPD solution did not converge')

        retdict = {'times':times,
                   'mags':np.full_like(mags, np.nan),
                   'errs':errs,
                   'fitcoeffs':coeffs,
                   'residuals':residuals}

        return retdict

Detrends a magnitude series given in mag using accompanying values of S in fsv D in fdv K in fkv x coords in xcc y coords in ycc background in bgv and background error in bge . smooth is used to set a smoothing parameter for the fit function . Does EPD voodoo .


def _epd_function(coeffs, fsv, fdv, fkv, xcc, ycc, bgv, bge, iha, izd):
    '''
    This is the EPD function to fit using a smoothed mag-series.

    '''

    return (coeffs[0]*fsv*fsv +
            coeffs[1]*fsv +
            coeffs[2]*fdv*fdv +
            coeffs[3]*fdv +
            coeffs[4]*fkv*fkv +
            coeffs[5]*fkv +
            coeffs[6] +
            coeffs[7]*fsv*fdv +
            coeffs[8]*fsv*fkv +
            coeffs[9]*fdv*fkv +
            coeffs[10]*np.sin(2*pi_value*xcc) +
            coeffs[11]*np.cos(2*pi_value*xcc) +
            coeffs[12]*np.sin(2*pi_value*ycc) +
            coeffs[13]*np.cos(2*pi_value*ycc) +
            coeffs[14]*np.sin(4*pi_value*xcc) +
            coeffs[15]*np.cos(4*pi_value*xcc) +
            coeffs[16]*np.sin(4*pi_value*ycc) +
            coeffs[17]*np.cos(4*pi_value*ycc) +
            coeffs[18]*bgv +
            coeffs[19]*bge +
            coeffs[20]*iha +
            coeffs[21]*izd)

def _epd_function ( coeffs , fsv , fdv , fkv , xcc , ycc , bgv , bge , iha , izd ) : return ( coeffs [ 0 ] * fsv * fsv + coeffs [ 1 ] * fsv + coeffs [ 2 ] * fdv * fdv + coeffs [ 3 ] * fdv + coeffs [ 4 ] * fkv * fkv + coeffs [ 5 ] * fkv + coeffs [ 6 ] + coeffs [ 7 ] * fsv * fdv + coeffs [ 8 ] * fsv * fkv + coeffs [ 9 ] * fdv * fkv + coeffs [ 10 ] * np . sin ( 2 * pi_value * xcc ) + coeffs [ 11 ] * np . cos ( 2 * pi_value * xcc ) + coeffs [ 12 ] * np . sin ( 2 * pi_value * ycc ) + coeffs [ 13 ] * np . cos ( 2 * pi_value * ycc ) + coeffs [ 14 ] * np . sin ( 4 * pi_value * xcc ) + coeffs [ 15 ] * np . cos ( 4 * pi_value * xcc ) + coeffs [ 16 ] * np . sin ( 4 * pi_value * ycc ) + coeffs [ 17 ] * np . cos ( 4 * pi_value * ycc ) + coeffs [ 18 ] * bgv + coeffs [ 19 ] * bge + coeffs [ 20 ] * iha + coeffs [ 21 ] * izd )

This is the EPD function to fit using a smoothed mag - series .


def _epd_residual2(coeffs,
                   times, mags, errs,
                   fsv, fdv, fkv, xcc, ycc, bgv, bge, iha, izd):
    '''This is the residual function to minimize using
    scipy.optimize.least_squares.

    This variant is for :py:func:`.epd_magseries_extparams`.

    '''

    f = _epd_function(coeffs, fsv, fdv, fkv, xcc, ycc, bgv, bge, iha, izd)
    residual = mags - f
    return residual

def _epd_residual2 ( coeffs , times , mags , errs , fsv , fdv , fkv , xcc , ycc , bgv , bge , iha , izd ) : f = _epd_function ( coeffs , fsv , fdv , fkv , xcc , ycc , bgv , bge , iha , izd ) residual = mags - f return residual

This is the residual function to minimize using scipy . optimize . least_squares .


def epd_magseries(times, mags, errs,
                  fsv, fdv, fkv, xcc, ycc, bgv, bge, iha, izd,
                  magsarefluxes=False,
                  epdsmooth_sigclip=3.0,
                  epdsmooth_windowsize=21,
                  epdsmooth_func=smooth_magseries_savgol,
                  epdsmooth_extraparams=None):
    '''Detrends a magnitude series using External Parameter Decorrelation.

    Requires a set of external parameters similar to those present in HAT light
    curves. At the moment, the HAT light-curve-specific external parameters are:

    - S: the 'fsv' column in light curves,
    - D: the 'fdv' column in light curves,
    - K: the 'fkv' column in light curves,
    - x coords: the 'xcc' column in light curves,
    - y coords: the 'ycc' column in light curves,
    - background value: the 'bgv' column in light curves,
    - background error: the 'bge' column in light curves,
    - hour angle: the 'iha' column in light curves,
    - zenith distance: the 'izd' column in light curves

    S, D, and K are defined as follows:

    - S -> measure of PSF sharpness (~1/sigma^2 sosmaller S = wider PSF)
    - D -> measure of PSF ellipticity in xy direction
    - K -> measure of PSF ellipticity in cross direction

    S, D, K are related to the PSF's variance and covariance, see eqn 30-33 in
    A. Pal's thesis: https://arxiv.org/abs/0906.3486

    NOTE: The errs are completely ignored and returned unchanged (except for
    sigclip and finite filtering).

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to detrend.

    fsv : np.array
        Array containing the external parameter `S` of the same length as times.

    fdv : np.array
        Array containing the external parameter `D` of the same length as times.

    fkv : np.array
        Array containing the external parameter `K` of the same length as times.

    xcc : np.array
        Array containing the external parameter `x-coords` of the same length as
        times.

    ycc : np.array
        Array containing the external parameter `y-coords` of the same length as
        times.

    bgv : np.array
        Array containing the external parameter `background value` of the same
        length as times.

    bge : np.array
        Array containing the external parameter `background error` of the same
        length as times.

    iha : np.array
        Array containing the external parameter `hour angle` of the same length
        as times.

    izd : np.array
        Array containing the external parameter `zenith distance` of the same
        length as times.

    magsarefluxes : bool
        Set this to True if `mags` actually contains fluxes.

    epdsmooth_sigclip : float or int or sequence of two floats/ints or None
        This specifies how to sigma-clip the input LC before fitting the EPD
        function to it.

        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    epdsmooth_windowsize : int
        This is the number of LC points to smooth over to generate a smoothed
        light curve that will be used to fit the EPD function.

    epdsmooth_func : Python function
        This sets the smoothing filter function to use. A Savitsky-Golay filter
        is used to smooth the light curve by default. The functions that can be
        used with this kwarg are listed in `varbase.trends`. If you want to use
        your own function, it MUST have the following signature::

                def smoothfunc(mags_array, window_size, **extraparams)

        and return a numpy array of the same size as `mags_array` with the
        smoothed time-series. Any extra params can be provided using the
        `extraparams` dict.

    epdsmooth_extraparams : dict
        This is a dict of any extra filter params to supply to the smoothing
        function.

    Returns
    -------

    dict
        Returns a dict of the following form::

            {'times':the input times after non-finite elems removed,
             'mags':the EPD detrended mag values (the EPD mags),
             'errs':the errs after non-finite elems removed,
             'fitcoeffs':EPD fit coefficient values,
             'fitinfo':the full tuple returned by scipy.leastsq,
             'fitmags':the EPD fit function evaluated at times,
             'mags_median': this is median of the EPD mags,
             'mags_mad': this is the MAD of EPD mags}

    '''

    finind = np.isfinite(times) & np.isfinite(mags) & np.isfinite(errs)
    ftimes, fmags, ferrs = times[::][finind], mags[::][finind], errs[::][finind]
    ffsv, ffdv, ffkv, fxcc, fycc, fbgv, fbge, fiha, fizd = (
        fsv[::][finind],
        fdv[::][finind],
        fkv[::][finind],
        xcc[::][finind],
        ycc[::][finind],
        bgv[::][finind],
        bge[::][finind],
        iha[::][finind],
        izd[::][finind],
    )

    stimes, smags, serrs, separams = sigclip_magseries_with_extparams(
        times, mags, errs,
        [fsv, fdv, fkv, xcc, ycc, bgv, bge, iha, izd],
        sigclip=epdsmooth_sigclip,
        magsarefluxes=magsarefluxes
    )
    sfsv, sfdv, sfkv, sxcc, sycc, sbgv, sbge, siha, sizd = separams

    # smooth the signal
    if isinstance(epdsmooth_extraparams, dict):
        smoothedmags = epdsmooth_func(smags,
                                      epdsmooth_windowsize,
                                      **epdsmooth_extraparams)
    else:
        smoothedmags = epdsmooth_func(smags, epdsmooth_windowsize)

    # initial fit coeffs
    initcoeffs = np.zeros(22)

    # fit the smoothed mags and find the EPD function coefficients
    leastsqfit = leastsq(_epd_residual,
                         initcoeffs,
                         args=(smoothedmags,
                               sfsv, sfdv, sfkv, sxcc,
                               sycc, sbgv, sbge, siha, sizd),
                         full_output=True)

    # if the fit succeeds, then get the EPD mags
    if leastsqfit[-1] in (1,2,3,4):

        fitcoeffs = leastsqfit[0]
        epdfit = _epd_function(fitcoeffs,
                               ffsv, ffdv, ffkv, fxcc, fycc,
                               fbgv, fbge, fiha, fizd)

        epdmags = npmedian(fmags) + fmags - epdfit

        retdict = {'times':ftimes,
                   'mags':epdmags,
                   'errs':ferrs,
                   'fitcoeffs':fitcoeffs,
                   'fitinfo':leastsqfit,
                   'fitmags':epdfit,
                   'mags_median':npmedian(epdmags),
                   'mags_mad':npmedian(npabs(epdmags - npmedian(epdmags)))}

        return retdict

    # if the solution fails, return nothing
    else:

        LOGERROR('EPD fit did not converge')
        return None

Detrends a magnitude series using External Parameter Decorrelation .


def rfepd_magseries(times, mags, errs,
                    externalparam_arrs,
                    magsarefluxes=False,
                    epdsmooth=True,
                    epdsmooth_sigclip=3.0,
                    epdsmooth_windowsize=21,
                    epdsmooth_func=smooth_magseries_savgol,
                    epdsmooth_extraparams=None,
                    rf_subsample=1.0,
                    rf_ntrees=300,
                    rf_extraparams={'criterion':'mse',
                                    'oob_score':False,
                                    'n_jobs':-1}):
    '''This uses a `RandomForestRegressor` to de-correlate the given magseries.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to run EPD on.

    externalparam_arrs : list of np.arrays
        This is a list of ndarrays of external parameters to decorrelate
        against. These should all be the same size as `times`, `mags`, `errs`.

    epdsmooth : bool
        If True, sets the training LC for the RandomForestRegress to be a
        smoothed version of the sigma-clipped light curve provided in `times`,
        `mags`, `errs`.

    epdsmooth_sigclip : float or int or sequence of two floats/ints or None
        This specifies how to sigma-clip the input LC before smoothing it and
        fitting the EPD function to it. The actual LC will not be sigma-clipped.

        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    epdsmooth_windowsize : int
        This is the number of LC points to smooth over to generate a smoothed
        light curve that will be used to fit the EPD function.

    epdsmooth_func : Python function
        This sets the smoothing filter function to use. A Savitsky-Golay filter
        is used to smooth the light curve by default. The functions that can be
        used with this kwarg are listed in `varbase.trends`. If you want to use
        your own function, it MUST have the following signature::

                def smoothfunc(mags_array, window_size, **extraparams)

        and return a numpy array of the same size as `mags_array` with the
        smoothed time-series. Any extra params can be provided using the
        `extraparams` dict.

    epdsmooth_extraparams : dict
        This is a dict of any extra filter params to supply to the smoothing
        function.

    rf_subsample : float
        Defines the fraction of the size of the `mags` array to use for
        training the random forest regressor.

    rf_ntrees : int
        This is the number of trees to use for the `RandomForestRegressor`.

    rf_extraprams : dict
        This is a dict of any extra kwargs to provide to the
        `RandomForestRegressor` instance used.

    Returns
    -------

    dict
        Returns a dict with decorrelated mags and the usual info from the
        `RandomForestRegressor`: variable importances, etc.

    '''

    # get finite times, mags, errs
    finind = np.isfinite(times) & np.isfinite(mags) & np.isfinite(errs)
    ftimes, fmags, ferrs = times[::][finind], mags[::][finind], errs[::][finind]
    finalparam_arrs = []
    for ep in externalparam_arrs:
        finalparam_arrs.append(ep[::][finind])

    stimes, smags, serrs, eparams = sigclip_magseries_with_extparams(
        times, mags, errs,
        externalparam_arrs,
        sigclip=epdsmooth_sigclip,
        magsarefluxes=magsarefluxes
    )

    # smoothing is optional for RFR because we train on a fraction of the mag
    # series and so should not require a smoothed input to fit a function to
    if epdsmooth:

        # smooth the signal
        if isinstance(epdsmooth_extraparams, dict):
            smoothedmags = epdsmooth_func(smags,
                                          epdsmooth_windowsize,
                                          **epdsmooth_extraparams)
        else:
            smoothedmags = epdsmooth_func(smags,
                                          epdsmooth_windowsize)

    else:

        smoothedmags = smags

    # set up the regressor
    if isinstance(rf_extraparams, dict):
        RFR = RandomForestRegressor(n_estimators=rf_ntrees,
                                    **rf_extraparams)
    else:
        RFR = RandomForestRegressor(n_estimators=rf_ntrees)

    # collect the features
    features = np.column_stack(eparams)

    # fit, then generate the predicted values, then get corrected values

    # we fit on a randomly selected subsample of all the mags
    if rf_subsample < 1.0:
        featureindices = np.arange(smoothedmags.size)

        # these are sorted because time-order should be important
        training_indices = np.sort(
            npr.choice(featureindices,
                       size=int(rf_subsample*smoothedmags.size),
                       replace=False)
        )
    else:
        training_indices = np.arange(smoothedmags.size)

    RFR.fit(features[training_indices,:], smoothedmags[training_indices])

    # predict on the full feature set
    flux_corrections = RFR.predict(np.column_stack(finalparam_arrs))
    corrected_fmags = npmedian(fmags) + fmags - flux_corrections

    retdict = {'times':ftimes,
               'mags':corrected_fmags,
               'errs':ferrs,
               'feature_importances':RFR.feature_importances_,
               'regressor':RFR,
               'mags_median':npmedian(corrected_fmags),
               'mags_mad':npmedian(npabs(corrected_fmags -
                                         npmedian(corrected_fmags)))}

    return retdict

This uses a RandomForestRegressor to de - correlate the given magseries .


def stellingwerf_pdm_theta(times, mags, errs, frequency,
                           binsize=0.05, minbin=9):
    '''
    This calculates the Stellingwerf PDM theta value at a test frequency.

    Parameters
    ----------

    times,mags,errs : np.array
        The input time-series and associated errors.

    frequency : float
        The test frequency to calculate the theta statistic at.

    binsize : float
        The phase bin size to use.

    minbin : int
        The minimum number of items in a phase bin to consider in the
        calculation of the statistic.

    Returns
    -------

    theta_pdm : float
        The value of the theta statistic at the specified `frequency`.

    '''

    period = 1.0/frequency
    fold_time = times[0]

    phased = phase_magseries(times,
                             mags,
                             period,
                             fold_time,
                             wrap=False,
                             sort=True)

    phases = phased['phase']
    pmags = phased['mags']
    bins = nparange(0.0, 1.0, binsize)

    binnedphaseinds = npdigitize(phases, bins)

    binvariances = []
    binndets = []
    goodbins = 0

    for x in npunique(binnedphaseinds):

        thisbin_inds = binnedphaseinds == x
        thisbin_mags = pmags[thisbin_inds]

        if thisbin_mags.size > minbin:
            thisbin_variance = npvar(thisbin_mags,ddof=1)
            binvariances.append(thisbin_variance)
            binndets.append(thisbin_mags.size)
            goodbins = goodbins + 1

    # now calculate theta
    binvariances = nparray(binvariances)
    binndets = nparray(binndets)

    theta_top = npsum(binvariances*(binndets - 1)) / (npsum(binndets) -
                                                      goodbins)
    theta_bot = npvar(pmags,ddof=1)
    theta = theta_top/theta_bot

    return theta

def
stellingwerf_pdm_theta
(
times
,
mags
,
errs
,
frequency
,
binsize
=
0.05
,
minbin
=
9
)
:
period
=
1.0
/
frequency
fold_time
=
times
[
0
]
phased
=
phase_magseries
(
times
,
mags
,
period
,
fold_time
,
wrap
=
False
,
sort
=
True
)
phases
=
phased
[
'phase'
]
pmags
=
phased
[
'mags'
]
bins
=
nparange
(
0.0
,
1.0
,
binsize
)
binnedphaseinds
=
npdigitize
(
phases
,
bins
)
binvariances
=
[
]
binndets
=
[
]
goodbins
=
0
for
x
in
npunique
(
binnedphaseinds
)
:
thisbin_inds
=
binnedphaseinds
==
x
thisbin_mags
=
pmags
[
thisbin_inds
]
if
thisbin_mags
.
size
>
minbin
:
thisbin_variance
=
npvar
(
thisbin_mags
,
ddof
=
1
)
binvariances
.
append
(
thisbin_variance
)
binndets
.
append
(
thisbin_mags
.
size
)
goodbins
=
goodbins
+
1
# now calculate theta
binvariances
=
nparray
(
binvariances
)
binndets
=
nparray
(
binndets
)
theta_top
=
npsum
(
binvariances
*
(
binndets
1
)
)
/
(
npsum
(
binndets
)
goodbins
)
theta_bot
=
npvar
(
pmags
,
ddof
=
1
)
theta
=
theta_top
/
theta_bot
return
theta

This calculates the Stellingwerf PDM theta value at a test frequency .


def keplermag_to_sdssr(keplermag, kic_sdssg, kic_sdssr):
    '''Converts magnitude measurements in Kepler band to SDSS r band.

    Parameters
    ----------

    keplermag : float or array-like
        The Kepler magnitude value(s) to convert to fluxes.

    kic_sdssg,kic_sdssr : float or array-like
        The SDSS g and r magnitudes of the object(s) from the Kepler Input
        Catalog. The .llc.fits MAST light curve file for a Kepler object
        contains these values in the FITS extension 0 header.

    Returns
    -------

    float or array-like
        SDSS r band magnitude(s) converted from the Kepler band magnitude.

    '''
    kic_sdssgr = kic_sdssg - kic_sdssr

    if kic_sdssgr < 0.8:
        kepsdssr = (keplermag - 0.2*kic_sdssg)/0.8
    else:
        kepsdssr = (keplermag - 0.1*kic_sdssg)/0.9
    return kepsdssr

def keplermag_to_sdssr ( keplermag , kic_sdssg , kic_sdssr ) : kic_sdssgr = kic_sdssg - kic_sdssr if kic_sdssgr < 0.8 : kepsdssr = ( keplermag - 0.2 * kic_sdssg ) / 0.8 else : kepsdssr = ( keplermag - 0.1 * kic_sdssg ) / 0.9 return kepsdssr

Converts magnitude measurements in Kepler band to SDSS r band .


def kepler_lcdict_to_pkl(lcdict, outfile=None):
    '''This writes the `lcdict` to a Python pickle.

    Parameters
    ----------

    lcdict : lcdict
        This is the input `lcdict` to write to a pickle.

    outfile : str or None
        If this is None, the object's Kepler ID/EPIC ID will determined from the
        `lcdict` and used to form the filename of the output pickle file. If
        this is a `str`, the provided filename will be used.

    Returns
    -------

    str
        The absolute path to the written pickle file.

    '''

    if not outfile:
        outfile = '%s-keplc.pkl' % lcdict['objectid'].replace(' ','-')

    # we're using pickle.HIGHEST_PROTOCOL here, this will make Py3 pickles
    # unreadable for Python 2.7
    with open(outfile,'wb') as outfd:
        pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    return os.path.abspath(outfile)

This writes the lcdict to a Python pickle .


def read_kepler_pklc(picklefile):
    '''This turns the pickled lightcurve file back into an `lcdict`.

    Parameters
    ----------

    picklefile : str
        The path to a previously written Kepler LC picklefile generated by
        `kepler_lcdict_to_pkl` above.

    Returns
    -------

    lcdict
        Returns an `lcdict` (this is useable by most astrobase functions for LC
        processing).

    '''

    if picklefile.endswith('.gz'):
        infd = gzip.open(picklefile, 'rb')
    else:
        infd = open(picklefile, 'rb')

    try:
        with infd:
            lcdict = pickle.load(infd)

    except UnicodeDecodeError:

        with open(picklefile,'rb') as infd:
            lcdict = pickle.load(infd, encoding='latin1')

        LOGWARNING('pickle %s was probably from Python 2 '
                   'and failed to load without using "latin1" encoding. '
                   'This is probably a numpy issue: '
                   'http://stackoverflow.com/q/11305790' % picklefile)

    return lcdict

This turns the pickled lightcurve file back into an lcdict .


def filter_kepler_lcdict(lcdict,
                         filterflags=True,
                         nanfilter='sap,pdc',
                         timestoignore=None):
    '''This filters the Kepler `lcdict`, removing nans and bad
    observations.

    By default, this function removes points in the Kepler LC that have ANY
    quality flags set.

    Parameters
    ----------

    lcdict : lcdict
        An `lcdict` produced by `consolidate_kepler_fitslc` or
        `read_kepler_fitslc`.

    filterflags : bool
        If True, will remove any measurements that have non-zero quality flags
        present. This usually indicates an issue with the instrument or
        spacecraft.

    nanfilter : {'sap','pdc','sap,pdc'}
        Indicates the flux measurement type(s) to apply the filtering to.

    timestoignore : list of tuples or None
        This is of the form::

            [(time1_start, time1_end), (time2_start, time2_end), ...]

        and indicates the start and end times to mask out of the final
        lcdict. Use this to remove anything that wasn't caught by the quality
        flags.

    Returns
    -------

    lcdict
        Returns an `lcdict` (this is useable by most astrobase functions for LC
        processing). The `lcdict` is filtered IN PLACE!

    '''

    cols = lcdict['columns']

    # filter all bad LC points as noted by quality flags
    if filterflags:

        nbefore = lcdict['time'].size
        filterind = lcdict['sap_quality'] == 0

        for col in cols:
            if '.' in col:
                key, subkey = col.split('.')
                lcdict[key][subkey] = lcdict[key][subkey][filterind]
            else:
                lcdict[col] = lcdict[col][filterind]

        nafter = lcdict['time'].size
        LOGINFO('applied quality flag filter, ndet before = %s, ndet after = %s'
                % (nbefore, nafter))

    if nanfilter and nanfilter == 'sap,pdc':
        notnanind = (
            npisfinite(lcdict['sap']['sap_flux']) &
            npisfinite(lcdict['pdc']['pdcsap_flux']) &
            npisfinite(lcdict['time'])
        )
    elif nanfilter and nanfilter == 'sap':
        notnanind = (
            npisfinite(lcdict['sap']['sap_flux']) &
            npisfinite(lcdict['time'])
        )
    elif nanfilter and nanfilter == 'pdc':
        notnanind = (
            npisfinite(lcdict['pdc']['pdcsap_flux']) &
            npisfinite(lcdict['time'])
        )

    # remove nans from all columns
    if nanfilter:

        nbefore = lcdict['time'].size
        for col in cols:
            if '.' in col:
                key, subkey = col.split('.')
                lcdict[key][subkey] = lcdict[key][subkey][notnanind]
            else:
                lcdict[col] = lcdict[col][notnanind]

        nafter = lcdict['time'].size

        LOGINFO('removed nans, ndet before = %s, ndet after = %s'
                % (nbefore, nafter))

    # exclude all times in timestoignore
    if (timestoignore and
        isinstance(timestoignore, list) and
        len(timestoignore) > 0):

        exclind = npfull_like(lcdict['time'], True, dtype=np.bool_)
        nbefore = exclind.size

        # get all the masks
        for ignoretime in timestoignore:
            time0, time1 = ignoretime[0], ignoretime[1]
            thismask = ~((lcdict['time'] >= time0) & (lcdict['time'] <= time1))
            exclind = exclind & thismask

        # apply the masks
        for col in cols:
            if '.' in col:
                key, subkey = col.split('.')
                lcdict[key][subkey] = lcdict[key][subkey][exclind]
            else:
                lcdict[col] = lcdict[col][exclind]

        nafter = lcdict['time'].size
        LOGINFO('removed timestoignore, ndet before = %s, ndet after = %s'
                % (nbefore, nafter))

    return lcdict

This filters the Kepler lcdict removing nans and bad observations .


def _epd_function(coeffs, fluxes, xcc, ycc, bgv, bge):
    '''This is the EPD function to fit.

    Parameters
    ----------

    coeffs : array-like of floats
        Contains the EPD coefficients that will be used to generate the EPD fit
        function.

    fluxes : array-like
        The flux measurement array being used.

    xcc,ycc : array-like
        Arrays of the x and y coordinates associated with each measurement in
        `fluxes`.

    bgv,bge : array-like
        Arrays of the flux background value and the flux background error
        associated with each measurement in `fluxes`.

    Returns
    -------

    np.array
        Contains the fit function evaluated at each flux measurement value.

    '''

    epdf = (
        coeffs[0] +
        coeffs[1]*npsin(2*MPI*xcc) + coeffs[2]*npcos(2*MPI*xcc) +
        coeffs[3]*npsin(2*MPI*ycc) + coeffs[4]*npcos(2*MPI*ycc) +
        coeffs[5]*npsin(4*MPI*xcc) + coeffs[6]*npcos(4*MPI*xcc) +
        coeffs[7]*npsin(4*MPI*ycc) + coeffs[8]*npcos(4*MPI*ycc) +
        coeffs[9]*bgv +
        coeffs[10]*bge
    )

    return epdf

def _epd_function ( coeffs , fluxes , xcc , ycc , bgv , bge ) : epdf = ( coeffs [ 0 ] + coeffs [ 1 ] * npsin ( 2 * MPI * xcc ) + coeffs [ 2 ] * npcos ( 2 * MPI * xcc ) + coeffs [ 3 ] * npsin ( 2 * MPI * ycc ) + coeffs [ 4 ] * npcos ( 2 * MPI * ycc ) + coeffs [ 5 ] * npsin ( 4 * MPI * xcc ) + coeffs [ 6 ] * npcos ( 4 * MPI * xcc ) + coeffs [ 7 ] * npsin ( 4 * MPI * ycc ) + coeffs [ 8 ] * npcos ( 4 * MPI * ycc ) + coeffs [ 9 ] * bgv + coeffs [ 10 ] * bge ) return epdf


def get_centroid_offsets(lcd, t_ing_egr, oot_buffer_time=0.1, sample_factor=3):
    '''After running `detrend_centroid`, this gets positions of centroids during
    transits, and outside of transits.

    These positions can then be used in a false positive analysis.

    This routine requires knowing the ingress and egress times for every
    transit of interest within the quarter this routine is being called for.
    There is currently no astrobase routine that automates this for periodic
    transits (it must be done in a calling routine).

    To get out of transit centroids, this routine takes points outside of the
    "buffer" set by `oot_buffer_time`, sampling 3x as many points on either
    side of the transit as are in the transit (or however many are specified by
    `sample_factor`).

    Parameters
    ----------

    lcd : lcdict
        An `lcdict` generated by the `read_kepler_fitslc` function. We assume
        that the `detrend_centroid` function has been run on this `lcdict`.

    t_ing_egr : list of tuples
        This is of the form::

            [(ingress time of i^th transit, egress time of i^th transit)]

        for i the transit number index in this quarter (starts at zero at the
        beginning of every quarter). Assumes units of BJD.

    oot_buffer_time : float
        Number of days away from ingress and egress times to begin sampling "out
        of transit" centroid points. The number of out of transit points to take
        per transit is 3x the number of points in transit.

    sample_factor : float
        The size of out of transit window from which to sample.

    Returns
    -------

    dict
        This is a dictionary keyed by transit number (i.e., the same index as
        `t_ing_egr`), where each key contains the following value::

            {'ctd_x_in_tra':ctd_x_in_tra,
             'ctd_y_in_tra':ctd_y_in_tra,
             'ctd_x_oot':ctd_x_oot,
             'ctd_y_oot':ctd_y_oot,
             'npts_in_tra':len(ctd_x_in_tra),
             'npts_oot':len(ctd_x_oot),
             'in_tra_times':in_tra_times,
             'oot_times':oot_times}

    '''

    # NOTE:
    # Bryson+ (2013) gives a more complicated and more correct approach to this
    # problem, computing offsets relative to positions defined on the SKY. This
    # requires using a Kepler focal plane geometry model. I don't have that
    # model, or know how to get it. So I use a simpler approach.

    qnum = int(np.unique(lcd['quarter']))
    LOGINFO('Getting centroid offsets (qnum: {:d})...'.format(qnum))
    # Kepler pixel scale, cf.
    # https://keplerscience.arc.nasa.gov/the-kepler-space-telescope.html
    arcsec_per_px = 3.98

    # Get the residuals (units: pixel offset).
    times = lcd['ctd_dtr']['times']
    ctd_resid_x = lcd['ctd_dtr']['ctd_x'] - lcd['ctd_dtr']['fit_ctd_x']
    ctd_resid_y = lcd['ctd_dtr']['ctd_y'] - lcd['ctd_dtr']['fit_ctd_y']

    # Return results in "centroid dictionary" (has keys of transit number).
    cd = {}
    for ix,(t_ing,t_egr) in enumerate(t_ing_egr):

        # We have in-transit times as input.
        in_tra_times = times[(times > t_ing) & (times < t_egr)]

        # Compute out of transit times on either side of the in-transit times.
        transit_dur = t_egr - t_ing
        oot_window_len = sample_factor * transit_dur

        oot_before = times[
            (times < (t_ing-oot_buffer_time)) &
            (times > (t_ing-oot_buffer_time-oot_window_len))
        ]
        oot_after = times[
            (times > (t_egr+oot_buffer_time)) &
            (times < (t_egr+oot_buffer_time+oot_window_len))
        ]

        oot_times = npconcatenate([oot_before, oot_after])

        mask_tra = npin1d(times, in_tra_times)
        mask_oot = npin1d(times, oot_times)

        # Convert to units of arcseconds.
        ctd_x_in_tra = ctd_resid_x[mask_tra]*arcsec_per_px
        ctd_y_in_tra = ctd_resid_y[mask_tra]*arcsec_per_px
        ctd_x_oot = ctd_resid_x[mask_oot]*arcsec_per_px
        ctd_y_oot = ctd_resid_y[mask_oot]*arcsec_per_px

        cd[ix] = {'ctd_x_in_tra':ctd_x_in_tra,
                  'ctd_y_in_tra':ctd_y_in_tra,
                  'ctd_x_oot':ctd_x_oot,
                  'ctd_y_oot':ctd_y_oot,
                  'npts_in_tra':len(ctd_x_in_tra),
                  'npts_oot':len(ctd_x_oot),
                  'in_tra_times':in_tra_times,
                  'oot_times':oot_times}

    LOGINFO('Got centroid offsets (qnum: {:d}).'.format(qnum))

    return cd

After running detrend_centroid this gets positions of centroids during transits and outside of transits .


def _get_legendre_deg_ctd(npts):
    '''This is a helper function for centroid detrending.

    '''

    from scipy.interpolate import interp1d

    degs = nparray([4,5,6,10,15])
    pts = nparray([1e2,3e2,5e2,1e3,3e3])
    fn = interp1d(pts, degs, kind='linear',
                  bounds_error=False,
                  fill_value=(min(degs), max(degs)))
    legendredeg = int(npfloor(fn(npts)))

    return legendredeg

def _get_legendre_deg_ctd ( npts ) : from scipy . interpolate import interp1d degs = nparray ( [ 4 , 5 , 6 , 10 , 15 ] ) pts = nparray ( [ 1e2 , 3e2 , 5e2 , 1e3 , 3e3 ] ) fn = interp1d ( pts , degs , kind = 'linear' , bounds_error = False , fill_value = ( min ( degs ) , max ( degs ) ) ) legendredeg = int ( npfloor ( fn ( npts ) ) ) return legendredeg

This is a helper function for centroid detrending .


def _legendre_dtr(x, y, y_err, legendredeg=10):
    '''This calculates the residual and chi-sq values for a Legendre
    function fit.

    Parameters
    ----------

    x : np.array
        Array of the independent variable.

    y : np.array
        Array of the dependent variable.

    y_err : np.array
        Array of errors associated with each `y` value. Used to calculate fit
        weights.

    legendredeg : int
        The degree of the Legendre function to use when fitting.

    Returns
    -------

    tuple
        The tuple returned is of the form: (fit_y, fitchisq, fitredchisq)

    '''
    try:
        p = Legendre.fit(x, y, legendredeg)
        fit_y = p(x)
    except Exception as e:
        fit_y = npzeros_like(y)

    fitchisq = npsum(
        ((fit_y - y)*(fit_y - y)) / (y_err*y_err)
    )

    nparams = legendredeg + 1
    fitredchisq = fitchisq/(len(y) - nparams - 1)

    LOGINFO(
        'legendre detrend applied. chisq = %.5f, reduced chisq = %.5f' %
        (fitchisq, fitredchisq)
    )

    return fit_y, fitchisq, fitredchisq

This calculates the residual and chi - sq values for a Legendre function fit .


def timebinlc(lcfile,
              binsizesec,
              outdir=None,
              lcformat='hat-sql',
              lcformatdir=None,
              timecols=None,
              magcols=None,
              errcols=None,
              minbinelems=7):

    '''This bins the given light curve file in time using the specified bin size.

    Parameters
    ----------

    lcfile : str
        The file name to process.

    binsizesec : float
        The time bin-size in seconds.

    outdir : str or None
        If this is a str, the output LC will be written to `outdir`. If this is
        None, the output LC will be written to the same directory as `lcfile`.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    Returns
    -------

    str
        The name of the output pickle file with the binned LC.

        Writes the output binned light curve to a pickle that contains the
        lcdict with an added `lcdict['binned'][magcol]` key, which contains the
        binned times, mags/fluxes, and errs as
        `lcdict['binned'][magcol]['times']`, `lcdict['binned'][magcol]['mags']`,
        and `lcdict['epd'][magcol]['errs']` for each `magcol` provided in the
        input or default `magcols` value for this light curve format.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # override the default timecols, magcols, and errcols
    # using the ones provided to the function
    if timecols is None:
        timecols = dtimecols
    if magcols is None:
        magcols = dmagcols
    if errcols is None:
        errcols = derrcols

    # get the LC into a dict
    lcdict = readerfunc(lcfile)

    # this should handle lists/tuples being returned by readerfunc
    # we assume that the first element is the actual lcdict
    # FIXME: figure out how to not need this assumption
    if ( (isinstance(lcdict, (list, tuple))) and
         (isinstance(lcdict[0], dict)) ):
        lcdict = lcdict[0]

    # skip already binned light curves
    if 'binned' in lcdict:
        LOGERROR('this light curve appears to be binned already, skipping...')
        return None

    lcdict['binned'] = {}

    for tcol, mcol, ecol in zip(timecols, magcols, errcols):

        # dereference the columns and get them from the lcdict
        if '.' in tcol:
            tcolget = tcol.split('.')
        else:
            tcolget = [tcol]
        times = _dict_get(lcdict, tcolget)

        if '.' in mcol:
            mcolget = mcol.split('.')
        else:
            mcolget = [mcol]
        mags = _dict_get(lcdict, mcolget)

        if '.' in ecol:
            ecolget = ecol.split('.')
        else:
            ecolget = [ecol]
        errs = _dict_get(lcdict, ecolget)

        # normalize here if not using special normalization
        if normfunc is None:
            ntimes, nmags = normalize_magseries(
                times, mags,
                magsarefluxes=magsarefluxes
            )

            times, mags, errs = ntimes, nmags, errs

        # now bin the mag series as requested
        binned = time_bin_magseries_with_errs(times,
                                              mags,
                                              errs,
                                              binsize=binsizesec,
                                              minbinelems=minbinelems)

        # put this into the special binned key of the lcdict
        lcdict['binned'][mcol] = {'times':binned['binnedtimes'],
                                  'mags':binned['binnedmags'],
                                  'errs':binned['binnederrs'],
                                  'nbins':binned['nbins'],
                                  'timebins':binned['jdbins'],
                                  'binsizesec':binsizesec}

    # done with binning for all magcols, now generate the output file
    # this will always be a pickle

    if outdir is None:
        outdir = os.path.dirname(lcfile)

    outfile = os.path.join(outdir, '%s-binned%.1fsec-%s.pkl' %
                           (squeeze(lcdict['objectid']).replace(' ','-'),
                            binsizesec, lcformat))

    with open(outfile, 'wb') as outfd:
        pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    return outfile

This bins the given light curve file in time using the specified bin size .


def parallel_timebin(lclist,
                     binsizesec,
                     maxobjects=None,
                     outdir=None,
                     lcformat='hat-sql',
                     lcformatdir=None,
                     timecols=None,
                     magcols=None,
                     errcols=None,
                     minbinelems=7,
                     nworkers=NCPUS,
                     maxworkertasks=1000):
    '''This time-bins all the LCs in the list using the specified bin size.

    Parameters
    ----------

    lclist : list of str
        The input LCs to process.

    binsizesec : float
        The time bin size to use in seconds.

    maxobjects : int or None
        If provided, LC processing will stop at `lclist[maxobjects]`.

    outdir : str or None
        The directory where output LCs will be written. If None, will write to
        the same directory as the input LCs.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    nworkers : int
        Number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before being
        replaced to guard against memory leaks.

    Returns
    -------

    dict
        The returned dict contains keys = input LCs, vals = output LCs.

    '''

    if outdir and not os.path.exists(outdir):
        os.mkdir(outdir)

    if maxobjects is not None:
        lclist = lclist[:maxobjects]

    tasks = [(x, binsizesec, {'outdir':outdir,
                              'lcformat':lcformat,
                              'lcformatdir':lcformatdir,
                              'timecols':timecols,
                              'magcols':magcols,
                              'errcols':errcols,
                              'minbinelems':minbinelems}) for x in lclist]

    pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)
    results = pool.map(timebinlc_worker, tasks)
    pool.close()
    pool.join()

    resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}

    return resdict

def parallel_timebin ( lclist , binsizesec , maxobjects = None , outdir = None , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , minbinelems = 7 , nworkers = NCPUS , maxworkertasks = 1000 ) : if outdir and not os . path . exists ( outdir ) : os . mkdir ( outdir ) if maxobjects is not None : lclist = lclist [ : maxobjects ] tasks = [ ( x , binsizesec , { 'outdir' : outdir , 'lcformat' : lcformat , 'lcformatdir' : lcformatdir , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'minbinelems' : minbinelems } ) for x in lclist ] pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks ) results = pool . map ( timebinlc_worker , tasks ) pool . close ( ) pool . join ( ) resdict = { os . path . basename ( x ) : y for ( x , y ) in zip ( lclist , results ) } return resdict

This time - bins all the LCs in the list using the specified bin size .


def parallel_timebin_lcdir(lcdir,
                           binsizesec,
                           maxobjects=None,
                           outdir=None,
                           lcformat='hat-sql',
                           lcformatdir=None,
                           timecols=None,
                           magcols=None,
                           errcols=None,
                           minbinelems=7,
                           nworkers=NCPUS,
                           maxworkertasks=1000):
    '''
    This time bins all the light curves in the specified directory.

    Parameters
    ----------

    lcdir : list of str
        Directory containing the input LCs to process.

    binsizesec : float
        The time bin size to use in seconds.

    maxobjects : int or None
        If provided, LC processing will stop at `lclist[maxobjects]`.

    outdir : str or None
        The directory where output LCs will be written. If None, will write to
        the same directory as the input LCs.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    nworkers : int
        Number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before being
        replaced to guard against memory leaks.

    Returns
    -------

    dict
        The returned dict contains keys = input LCs, vals = output LCs.

    '''
    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (fileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    lclist = sorted(glob.glob(os.path.join(lcdir, fileglob)))

    return parallel_timebin(lclist,
                            binsizesec,
                            maxobjects=maxobjects,
                            outdir=outdir,
                            lcformat=lcformat,
                            timecols=timecols,
                            magcols=magcols,
                            errcols=errcols,
                            minbinelems=minbinelems,
                            nworkers=nworkers,
                            maxworkertasks=maxworkertasks)

This time bins all the light curves in the specified directory .


def _varfeatures_worker(task):
    '''
    This wraps varfeatures.

    '''

    try:
        (lcfile, outdir, timecols, magcols, errcols,
         mindet, lcformat, lcformatdir) = task
        return get_varfeatures(lcfile, outdir,
                               timecols=timecols,
                               magcols=magcols,
                               errcols=errcols,
                               mindet=mindet,
                               lcformat=lcformat,
                               lcformatdir=lcformatdir)

    except Exception as e:
        return None

def _varfeatures_worker ( task ) : try : ( lcfile , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) = task return get_varfeatures ( lcfile , outdir , timecols = timecols , magcols = magcols , errcols = errcols , mindet = mindet , lcformat = lcformat , lcformatdir = lcformatdir ) except Exception as e : return None


def serial_varfeatures(lclist,
                       outdir,
                       maxobjects=None,
                       timecols=None,
                       magcols=None,
                       errcols=None,
                       mindet=1000,
                       lcformat='hat-sql',
                       lcformatdir=None):
    '''This runs variability feature extraction for a list of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        List of the generated variability features pickles for the input LCs,
        with results for each magcol in the input `magcol` or light curve
        format's default `magcol` list.

    '''

    if maxobjects:
        lclist = lclist[:maxobjects]

    tasks = [(x, outdir, timecols, magcols, errcols,
              mindet, lcformat, lcformatdir)
             for x in lclist]

    for task in tqdm(tasks):
        result = _varfeatures_worker(task)

    return result

def serial_varfeatures ( lclist , outdir , maxobjects = None , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None ) : if maxobjects : lclist = lclist [ : maxobjects ] tasks = [ ( x , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) for x in lclist ] for task in tqdm ( tasks ) : result = _varfeatures_worker ( task ) return result

This runs variability feature extraction for a list of LCs .


def parallel_varfeatures(lclist,
                         outdir,
                         maxobjects=None,
                         timecols=None,
                         magcols=None,
                         errcols=None,
                         mindet=1000,
                         lcformat='hat-sql',
                         lcformatdir=None,
                         nworkers=NCPUS):
    '''This runs variable feature extraction in parallel for all LCs in `lclist`.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.

    '''
    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        lclist = lclist[:maxobjects]

    tasks = [(x, outdir, timecols, magcols, errcols, mindet,
              lcformat, lcformatdir) for x in lclist]

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(varfeatures_worker, tasks)

    results = [x for x in resultfutures]
    resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}

    return resdict

This runs variable feature extraction in parallel for all LCs in lclist .


def parallel_varfeatures_lcdir(lcdir,
                               outdir,
                               fileglob=None,
                               maxobjects=None,
                               timecols=None,
                               magcols=None,
                               errcols=None,
                               recursive=True,
                               mindet=1000,
                               lcformat='hat-sql',
                               lcformatdir=None,
                               nworkers=NCPUS):
    '''This runs parallel variable feature extraction for a directory of LCs.

    Parameters
    ----------

    lcdir : str
        The directory of light curve files to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    fileglob : str or None
        The file glob to use when looking for light curve files in `lcdir`. If
        None, the default file glob associated for this LC format will be used.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    if not fileglob:
        fileglob = dfileglob

    # now find the files
    LOGINFO('searching for %s light curves in %s ...' % (lcformat, lcdir))

    if recursive is False:
        matching = glob.glob(os.path.join(lcdir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcdir,
                                              '**',
                                              fileglob),
                                 recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcdir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        LOGINFO('found %s light curves, getting varfeatures...' %
                len(matching))

        return parallel_varfeatures(matching,
                                    outdir,
                                    maxobjects=maxobjects,
                                    timecols=timecols,
                                    magcols=magcols,
                                    errcols=errcols,
                                    mindet=mindet,
                                    lcformat=lcformat,
                                    lcformatdir=lcformatdir,
                                    nworkers=nworkers)

    else:

        LOGERROR('no light curve files in %s format found in %s' % (lcformat,
                                                                    lcdir))
        return None

This runs parallel variable feature extraction for a directory of LCs .


def cp2png(checkplotin, extrarows=None):
    '''This is just a shortened form of the function above for convenience.

    This only handles pickle files as input.

    Parameters
    ----------

    checkplotin : str
        File name of a checkplot pickle file to convert to a PNG.

    extrarows : list of tuples
        This is a list of 4-element tuples containing paths to PNG files that
        will be added to the end of the rows generated from the checkplotin
        pickle/dict. Each tuple represents a row in the final output PNG
        file. If there are less than 4 elements per tuple, the missing elements
        will be filled in with white-space. If there are more than 4 elements
        per tuple, only the first four will be used.

        The purpose of this kwarg is to incorporate periodograms and phased LC
        plots (in the form of PNGs) generated from an external period-finding
        function or program (like VARTOOLS) to allow for comparison with
        astrobase results.

        NOTE: the PNG files specified in `extrarows` here will be added to those
        already present in the input `checkplotdict['externalplots']` if that is
        None because you passed in a similar list of external plots to the
        :py:func:`astrobase.checkplot.pkl.checkplot_pickle` function earlier. In
        this case, `extrarows` can be used to add even more external plots if
        desired.

        Each external plot PNG will be resized to 750 x 480 pixels to fit into
        an output image cell.

        By convention, each 4-element tuple should contain:

        - a periodiogram PNG
        - phased LC PNG with 1st best peak period from periodogram
        - phased LC PNG with 2nd best peak period from periodogram
        - phased LC PNG with 3rd best peak period from periodogram

        Example of extrarows::

            [('/path/to/external/bls-periodogram.png',
              '/path/to/external/bls-phasedlc-plot-bestpeak.png',
              '/path/to/external/bls-phasedlc-plot-peak2.png',
              '/path/to/external/bls-phasedlc-plot-peak3.png'),
             ('/path/to/external/pdm-periodogram.png',
              '/path/to/external/pdm-phasedlc-plot-bestpeak.png',
              '/path/to/external/pdm-phasedlc-plot-peak2.png',
              '/path/to/external/pdm-phasedlc-plot-peak3.png'),
            ...]

    Returns
    -------

    str
        The absolute path to the generated checkplot PNG.

    '''

    if checkplotin.endswith('.gz'):
        outfile = checkplotin.replace('.pkl.gz','.png')
    else:
        outfile = checkplotin.replace('.pkl','.png')

    return checkplot_pickle_to_png(checkplotin, outfile, extrarows=extrarows)

def cp2png ( checkplotin , extrarows = None ) : if checkplotin . endswith ( '.gz' ) : outfile = checkplotin . replace ( '.pkl.gz' , '.png' ) else : outfile = checkplotin . replace ( '.pkl' , '.png' ) return checkplot_pickle_to_png ( checkplotin , outfile , extrarows = extrarows )

This is just a shortened form of the function above for convenience .


def flare_model(flareparams, times, mags, errs):
    '''This is a flare model function, similar to Kowalski+ 2011.

    From the paper by Pitkin+ 2014:
    http://adsabs.harvard.edu/abs/2014MNRAS.445.2268P

    Parameters
    ----------

    flareparams : list of float
        This defines the flare model::

            [amplitude,
             flare_peak_time,
             rise_gaussian_stdev,
             decay_time_constant]

        where:

        `amplitude`: the maximum flare amplitude in mags or flux. If flux, then
        amplitude should be positive. If mags, amplitude should be negative.

        `flare_peak_time`: time at which the flare maximum happens.

        `rise_gaussian_stdev`: the stdev of the gaussian describing the rise of
        the flare.

        `decay_time_constant`: the time constant of the exponential fall of the
        flare.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate
        model mags.

    Returns
    -------

    (modelmags, times, mags, errs) : tuple
        Returns the model mags evaluated at the input time values. Also returns
        the input `times`, `mags`, and `errs`.

    '''

    (amplitude, flare_peak_time,
     rise_gaussian_stdev, decay_time_constant) = flareparams

    zerolevel = np.median(mags)
    modelmags = np.full_like(times, zerolevel)

    # before peak gaussian rise...
    modelmags[times < flare_peak_time] = (
        mags[times < flare_peak_time] +
        amplitude * np.exp(
            -((times[times < flare_peak_time] -
               flare_peak_time) *
              (times[times < flare_peak_time] -
               flare_peak_time)) /
            (2.0*rise_gaussian_stdev*rise_gaussian_stdev)
        )
    )

    # after peak exponential decay...
    modelmags[times > flare_peak_time] = (
        mags[times > flare_peak_time] +
        amplitude * np.exp(
            -((times[times > flare_peak_time] -
               flare_peak_time)) /
            (decay_time_constant)
        )
    )

    return modelmags, times, mags, errs

def
flare_model
(
flareparams
,
times
,
mags
,
errs
)
:
(
amplitude
,
flare_peak_time
,
rise_gaussian_stdev
,
decay_time_constant
)
=
flareparams
zerolevel
=
np
.
median
(
mags
)
modelmags
=
np
.
full_like
(
times
,
zerolevel
)
# before peak gaussian rise...
modelmags
[
times
<
flare_peak_time
]
=
(
mags
[
times
<
flare_peak_time
]
+
amplitude
*
np
.
exp
(
(
(
times
[
times
<
flare_peak_time
]
flare_peak_time
)
*
(
times
[
times
<
flare_peak_time
]
flare_peak_time
)
)
/
(
2.0
*
rise_gaussian_stdev
*
rise_gaussian_stdev
)
)
)
# after peak exponential decay...
modelmags
[
times
>
flare_peak_time
]
=
(
mags
[
times
>
flare_peak_time
]
+
amplitude
*
np
.
exp
(
(
(
times
[
times
>
flare_peak_time
]
flare_peak_time
)
)
/
(
decay_time_constant
)
)
)
return
modelmags
,
times
,
mags
,
errs

This is a flare model function similar to Kowalski + 2011 .


def flare_model_residual(flareparams, times, mags, errs):
    '''
    This returns the residual between model mags and the actual mags.

    Parameters
    ----------

    flareparams : list of float
        This defines the flare model::

            [amplitude,
             flare_peak_time,
             rise_gaussian_stdev,
             decay_time_constant]

        where:

        `amplitude`: the maximum flare amplitude in mags or flux. If flux, then
        amplitude should be positive. If mags, amplitude should be negative.

        `flare_peak_time`: time at which the flare maximum happens.

        `rise_gaussian_stdev`: the stdev of the gaussian describing the rise of
        the flare.

        `decay_time_constant`: the time constant of the exponential fall of the
        flare.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate
        model mags.

    Returns
    -------

    np.array
        The residuals between the input `mags` and generated `modelmags`,
        weighted by the measurement errors in `errs`.

    '''

    modelmags, _, _, _ = flare_model(flareparams, times, mags, errs)

    return (mags - modelmags)/errs

def flare_model_residual ( flareparams , times , mags , errs ) : modelmags , _ , _ , _ = flare_model ( flareparams , times , mags , errs ) return ( mags - modelmags ) / errs

This returns the residual between model mags and the actual mags .


def shutdown_check_handler():
    """This checks the AWS instance data URL to see if there's a pending
    shutdown for the instance.

    This is useful for AWS spot instances. If there is a pending shutdown posted
    to the instance data URL, we'll use the result of this function break out of
    the processing loop and shut everything down ASAP before the instance dies.

    Returns
    -------

    bool
        - True if the instance is going to die soon.
        - False if the instance is still safe.

    """

    url = 'http://169.254.169.254/latest/meta-data/spot/instance-action'

    try:
        resp = requests.get(url, timeout=1.0)
        resp.raise_for_status()

        stopinfo = resp.json()
        if 'action' in stopinfo and stopinfo['action'] in ('stop',
                                                           'terminate',
                                                           'hibernate'):
            stoptime = stopinfo['time']
            LOGWARNING('instance is going to %s at %s' % (stopinfo['action'],
                                                          stoptime))

            resp.close()
            return True
        else:
            resp.close()
            return False

    except HTTPError as e:
        resp.close()
        return False

    except Exception as e:
        resp.close()
        return False

This checks the AWS instance data URL to see if there s a pending shutdown for the instance .


def runcp_producer_loop_savedstate(
        use_saved_state=None,
        lightcurve_list=None,
        input_queue=None,
        input_bucket=None,
        result_queue=None,
        result_bucket=None,
        pfresult_list=None,
        runcp_kwargs=None,
        process_list_slice=None,
        download_when_done=True,
        purge_queues_when_done=True,
        save_state_when_done=True,
        delete_queues_when_done=False,
        s3_client=None,
        sqs_client=None
):
    """This wraps the function above to allow for loading previous state from a
    file.

    Parameters
    ----------

    use_saved_state : str or None
        This is the path to the saved state pickle file produced by a previous
        run of `runcp_producer_loop`. Will get all of the arguments to run
        another instance of the loop from that pickle file. If this is None, you
        MUST provide all of the appropriate arguments to that function.

    lightcurve_list : str or list of str or None
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str or None
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str or None
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str or None
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str or None
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict or None
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list or None
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool or None
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool or None
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.

    """

    if use_saved_state is not None and os.path.exists(use_saved_state):

        with open(use_saved_state,'rb') as infd:
            saved_state = pickle.load(infd)

        # run the producer loop using the saved state's todo list
        return runcp_producer_loop(
            saved_state['in_progress'],
            saved_state['args'][1],
            saved_state['args'][2],
            saved_state['args'][3],
            saved_state['args'][4],
            **saved_state['kwargs']
        )

    else:

        return runcp_producer_loop(
            lightcurve_list,
            input_queue,
            input_bucket,
            result_queue,
            result_bucket,
            pfresult_list=pfresult_list,
            runcp_kwargs=runcp_kwargs,
            process_list_slice=process_list_slice,
            download_when_done=download_when_done,
            purge_queues_when_done=purge_queues_when_done,
            save_state_when_done=save_state_when_done,
            delete_queues_when_done=delete_queues_when_done,
            s3_client=s3_client,
            sqs_client=sqs_client
        )

This wraps the function above to allow for loading previous state from a file .