def spline_fit_magseries(times, mags, errs, period,
                         knotfraction=0.01,
                         maxknots=30,
                         sigclip=30.0,
                         plotfit=False,
                         ignoreinitfail=False,
                         magsarefluxes=False,
                         verbose=True):

    '''This fits a univariate cubic spline to the phased light curve.

    This fit may be better than the Fourier fit for sharply variable objects,
    like EBs, so can be used to distinguish them from other types of variables.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to fit a spline to.

    period : float
        The period to use for the spline fit.

    knotfraction : float
        The knot fraction is the number of internal knots to use for the
        spline. A value of 0.01 (or 1%) of the total number of non-nan
        observations appears to work quite well, without over-fitting. maxknots
        controls the maximum number of knots that will be allowed.

    maxknots : int
        The maximum number of knots that will be used even if `knotfraction`
        gives a value to use larger than `maxknots`. This helps dealing with
        over-fitting to short time-scale variations.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    magsarefluxes : bool
        If True, will treat the input values of `mags` as fluxes for purposes of
        plotting the fit and sig-clipping.

    plotfit : str or False
        If this is a string, this function will make a plot for the fit to the
        mag/flux time-series and writes the plot to the path specified here.

    ignoreinitfail : bool
        If this is True, ignores the initial failure to find a set of optimized
        Fourier parameters using the global optimization function and proceeds
        to do a least-squares fit anyway.

    verbose : bool
        If True, will indicate progress and warn of any problems.

    Returns
    -------

    dict
        This function returns a dict containing the model fit parameters, the
        minimized chi-sq value and the reduced chi-sq value. The form of this
        dict is mostly standardized across all functions in this module::

            {
                'fittype':'spline',
                'fitinfo':{
                    'nknots': the number of knots used for the fit
                    'fitmags': the model fit mags,
                    'fitepoch': the epoch of minimum light for the fit,
                },
                'fitchisq': the minimized value of the fit's chi-sq,
                'fitredchisq':the reduced chi-sq value,
                'fitplotfile': the output fit plot if fitplot is not None,
                'magseries':{
                    'times':input times in phase order of the model,
                    'phase':the phases of the model mags,
                    'mags':input mags/fluxes in the phase order of the model,
                    'errs':errs in the phase order of the model,
                    'magsarefluxes':input value of magsarefluxes kwarg
                }
            }

    '''

    # this is required to fit the spline correctly
    if errs is None:
        errs = npfull_like(mags, 0.005)

    # sigclip the magnitude time series
    stimes, smags, serrs = sigclip_magseries(times, mags, errs,
                                             sigclip=sigclip,
                                             magsarefluxes=magsarefluxes)
    # get rid of zero errs
    nzind = npnonzero(serrs)
    stimes, smags, serrs = stimes[nzind], smags[nzind], serrs[nzind]

    # phase the mag series
    phase, pmags, perrs, ptimes, mintime = (
        get_phased_quantities(stimes, smags, serrs, period)
    )

    # now figure out the number of knots up to max knots (=100)
    nobs = len(phase)
    nknots = int(npfloor(knotfraction*nobs))
    nknots = maxknots if nknots > maxknots else nknots
    splineknots = nplinspace(phase[0] + 0.01,
                             phase[-1] - 0.01,
                             num=nknots)

    # NOTE: newer scipy needs x to be strictly increasing. this means we should
    # filter out anything that doesn't have np.diff(phase) > 0.0
    # FIXME: this needs to be tested
    phase_diffs_ind = npdiff(phase) > 0.0
    incphase_ind = npconcatenate((nparray([True]), phase_diffs_ind))
    phase, pmags, perrs = (phase[incphase_ind],
                           pmags[incphase_ind],
                           perrs[incphase_ind])

    # generate and fit the spline
    spl = LSQUnivariateSpline(phase, pmags, t=splineknots, w=1.0/perrs)

    # calculate the spline fit to the actual phases, the chisq and red-chisq
    fitmags = spl(phase)

    fitchisq = npsum(
        ((fitmags - pmags)*(fitmags - pmags)) / (perrs*perrs)
    )

    fitredchisq = fitchisq/(len(pmags) - nknots - 1)

    if verbose:
        LOGINFO(
            'spline fit done. nknots = %s,  '
            'chisq = %.5f, reduced chisq = %.5f' %
            (nknots, fitchisq, fitredchisq)
        )

    # figure out the time of light curve minimum (i.e. the fit epoch)
    # this is when the fit mag is maximum (i.e. the faintest)
    # or if magsarefluxes = True, then this is when fit flux is minimum
    if not magsarefluxes:
        fitmagminind = npwhere(fitmags == npmax(fitmags))
    else:
        fitmagminind = npwhere(fitmags == npmin(fitmags))
    if len(fitmagminind[0]) > 1:
        fitmagminind = (fitmagminind[0][0],)
    magseriesepoch = ptimes[fitmagminind]

    # assemble the returndict
    returndict = {
        'fittype':'spline',
        'fitinfo':{
            'nknots':nknots,
            'fitmags':fitmags,
            'fitepoch':magseriesepoch
        },
        'fitchisq':fitchisq,
        'fitredchisq':fitredchisq,
        'fitplotfile':None,
        'magseries':{
            'times':ptimes,
            'phase':phase,
            'mags':pmags,
            'errs':perrs,
            'magsarefluxes':magsarefluxes
        },
    }

    # make the fit plot if required
    if plotfit and isinstance(plotfit, str):

        make_fit_plot(phase, pmags, perrs, fitmags,
                      period, mintime, magseriesepoch,
                      plotfit,
                      magsarefluxes=magsarefluxes)

        returndict['fitplotfile'] = plotfit

    return returndict

This fits a univariate cubic spline to the phased light curve .


def runcp_worker(task):
    '''
    This is the worker for running checkplots.

    Parameters
    ----------

    task : tuple
        This is of the form: (pfpickle, outdir, lcbasedir, kwargs).

    Returns
    -------

    list of str
        The list of checkplot pickles returned by the `runcp` function.

    '''

    pfpickle, outdir, lcbasedir, kwargs = task

    try:

        return runcp(pfpickle, outdir, lcbasedir, **kwargs)

    except Exception as e:

        LOGEXCEPTION(' could not make checkplots for %s: %s' % (pfpickle, e))
        return None

def
runcp_worker
(
task
)
:
pfpickle
,
outdir
,
lcbasedir
,
kwargs
=
task
try
:
return
runcp
(
pfpickle
,
outdir
,
lcbasedir
,
*
*
kwargs
)
except
Exception
as
e
:
LOGEXCEPTION
(
' could not make checkplots for %s: %s'
%
(
pfpickle
,
e
)
)
return

This is the worker for running checkplots .


def parallel_cp(
        pfpicklelist,
        outdir,
        lcbasedir,
        fast_mode=False,
        lcfnamelist=None,
        cprenorm=False,
        lclistpkl=None,
        gaia_max_timeout=60.0,
        gaia_mirror=None,
        nbrradiusarcsec=60.0,
        maxnumneighbors=5,
        makeneighborlcs=True,
        xmatchinfo=None,
        xmatchradiusarcsec=3.0,
        sigclip=10.0,
        minobservations=99,
        lcformat='hat-sql',
        lcformatdir=None,
        timecols=None,
        magcols=None,
        errcols=None,
        skipdone=False,
        done_callback=None,
        done_callback_args=None,
        done_callback_kwargs=None,
        liststartindex=None,
        maxobjects=None,
        nworkers=NCPUS,
):
    '''This drives the parallel execution of `runcp` for a list of periodfinding
    result pickles.

    Parameters
    ----------

    pfpicklelist : list of str or list of Nones
        This is the list of the filenames of the period-finding result pickles
        to process. To make checkplots using the light curves directly, set this
        to a list of Nones with the same length as the list of light curve files
        that you provide in `lcfnamelist`.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    lcfnamelist : list of str or None
        If this is provided, it must be a list of the input light curve
        filenames to process. These can either be associated with each input
        period-finder result pickle, or can be provided standalone to make
        checkplots without phased LC plots in them. In the second case, you must
        set `pfpicklelist` to a list of Nones that matches the length of
        `lcfnamelist`.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    liststartindex : int
        The index of the `pfpicklelist` (and `lcfnamelist` if provided) to start
        working at.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles (and light curves if `lcfnamelist`
        is also provided) over several sessions or machines.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    # work around the Darwin segfault after fork if no network activity in
    # main thread bug: https://bugs.python.org/issue30385#msg293958
    if sys.platform == 'darwin':
        import requests
        requests.get('http://captive.apple.com/hotspot-detect.html')

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    # handle the start and end indices
    if (liststartindex is not None) and (maxobjects is None):
        pfpicklelist = pfpicklelist[liststartindex:]
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[liststartindex:]

    elif (liststartindex is None) and (maxobjects is not None):
        pfpicklelist = pfpicklelist[:maxobjects]
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[:maxobjects]

    elif (liststartindex is not None) and (maxobjects is not None):
        pfpicklelist = (
            pfpicklelist[liststartindex:liststartindex+maxobjects]
        )
        if lcfnamelist is not None:
            lcfnamelist = lcfnamelist[liststartindex:liststartindex+maxobjects]

    # if the lcfnamelist is not provided, create a dummy
    if lcfnamelist is None:
        lcfnamelist = [None]*len(pfpicklelist)

    tasklist = [(x, outdir, lcbasedir,
                 {'lcformat':lcformat,
                  'lcformatdir':lcformatdir,
                  'lcfname':y,
                  'timecols':timecols,
                  'magcols':magcols,
                  'errcols':errcols,
                  'lclistpkl':lclistpkl,
                  'gaia_max_timeout':gaia_max_timeout,
                  'gaia_mirror':gaia_mirror,
                  'nbrradiusarcsec':nbrradiusarcsec,
                  'maxnumneighbors':maxnumneighbors,
                  'makeneighborlcs':makeneighborlcs,
                  'xmatchinfo':xmatchinfo,
                  'xmatchradiusarcsec':xmatchradiusarcsec,
                  'sigclip':sigclip,
                  'minobservations':minobservations,
                  'skipdone':skipdone,
                  'cprenorm':cprenorm,
                  'fast_mode':fast_mode,
                  'done_callback':done_callback,
                  'done_callback_args':done_callback_args,
                  'done_callback_kwargs':done_callback_kwargs}) for
                x,y in zip(pfpicklelist, lcfnamelist)]

    resultfutures = []
    results = []

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(runcp_worker, tasklist)

    results = [x for x in resultfutures]

    executor.shutdown()
    return results

This drives the parallel execution of runcp for a list of periodfinding result pickles .


def parallel_cp_pfdir(pfpickledir,
                      outdir,
                      lcbasedir,
                      pfpickleglob='periodfinding-*.pkl*',
                      lclistpkl=None,
                      cprenorm=False,
                      nbrradiusarcsec=60.0,
                      maxnumneighbors=5,
                      makeneighborlcs=True,
                      fast_mode=False,
                      gaia_max_timeout=60.0,
                      gaia_mirror=None,
                      xmatchinfo=None,
                      xmatchradiusarcsec=3.0,
                      minobservations=99,
                      sigclip=10.0,
                      lcformat='hat-sql',
                      lcformatdir=None,
                      timecols=None,
                      magcols=None,
                      errcols=None,
                      skipdone=False,
                      done_callback=None,
                      done_callback_args=None,
                      done_callback_kwargs=None,
                      maxobjects=None,
                      nworkers=32):

    '''This drives the parallel execution of `runcp` for a directory of
    periodfinding pickles.

    Parameters
    ----------

    pfpickledir : str
        This is the directory containing all of the period-finding pickles to
        process.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    pkpickleglob : str
        This is a UNIX file glob to select period-finding result pickles in the
        specified `pfpickledir`.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    maxobjects : int
        The maximum number of objects to process in this run.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    pfpicklelist = sorted(glob.glob(os.path.join(pfpickledir, pfpickleglob)))

    LOGINFO('found %s period-finding pickles, running cp...' %
            len(pfpicklelist))

    return parallel_cp(pfpicklelist,
                       outdir,
                       lcbasedir,
                       fast_mode=fast_mode,
                       lclistpkl=lclistpkl,
                       nbrradiusarcsec=nbrradiusarcsec,
                       gaia_max_timeout=gaia_max_timeout,
                       gaia_mirror=gaia_mirror,
                       maxnumneighbors=maxnumneighbors,
                       makeneighborlcs=makeneighborlcs,
                       xmatchinfo=xmatchinfo,
                       xmatchradiusarcsec=xmatchradiusarcsec,
                       sigclip=sigclip,
                       minobservations=minobservations,
                       cprenorm=cprenorm,
                       maxobjects=maxobjects,
                       lcformat=lcformat,
                       lcformatdir=lcformatdir,
                       timecols=timecols,
                       magcols=magcols,
                       errcols=errcols,
                       skipdone=skipdone,
                       nworkers=nworkers,
                       done_callback=done_callback,
                       done_callback_args=done_callback_args,
                       done_callback_kwargs=done_callback_kwargs)

This drives the parallel execution of runcp for a directory of periodfinding pickles .


def _runpf_worker(task):
    '''
    This runs the runpf function.

    '''

    (lcfile, outdir, timecols, magcols, errcols, lcformat, lcformatdir,
     pfmethods, pfkwargs, getblssnr, sigclip, nworkers, minobservations,
     excludeprocessed) = task

    if os.path.exists(lcfile):
        pfresult = runpf(lcfile,
                         outdir,
                         timecols=timecols,
                         magcols=magcols,
                         errcols=errcols,
                         lcformat=lcformat,
                         lcformatdir=lcformatdir,
                         pfmethods=pfmethods,
                         pfkwargs=pfkwargs,
                         getblssnr=getblssnr,
                         sigclip=sigclip,
                         nworkers=nworkers,
                         minobservations=minobservations,
                         excludeprocessed=excludeprocessed)
        return pfresult
    else:
        LOGERROR('LC does not exist for requested file %s' % lcfile)
        return None


def parallel_pf(lclist,
                outdir,
                timecols=None,
                magcols=None,
                errcols=None,
                lcformat='hat-sql',
                lcformatdir=None,
                pfmethods=('gls','pdm','mav','win'),
                pfkwargs=({},{},{},{}),
                sigclip=10.0,
                getblssnr=False,
                nperiodworkers=NCPUS,
                ncontrolworkers=1,
                liststartindex=None,
                listmaxobjects=None,
                minobservations=500,
                excludeprocessed=True):
    '''This drives the overall parallel period processing for a list of LCs.

    As a rough benchmark, 25000 HATNet light curves with up to 50000 points per
    LC take about 26 days in total for an invocation of this function using
    GLS+PDM+BLS, 10 periodworkers, and 4 controlworkers (so all 40 'cores') on a
    2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file to process.

    outdir : str
        The output directory where the period-finding result pickles will go.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    pfmethods : list of str
        This is a list of period finding methods to run. Each element is a
        string matching the keys of the `PFMETHODS` dict above. By default, this
        runs GLS, PDM, AoVMH, and the spectral window Lomb-Scargle periodogram.

    pfkwargs : list of dicts
        This is used to provide any special kwargs as dicts to each
        period-finding method function specified in `pfmethods`.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    getblssnr : bool
        If this is True and BLS is one of the methods specified in `pfmethods`,
        will also calculate the stats for each best period in the BLS results:
        transit depth, duration, ingress duration, refit period and epoch, and
        the SNR of the transit.

    nperiodworkers : int
        The number of parallel period-finding workers to launch per object task.

    ncontrolworkers : int
        The number of controlling processes to launch. This effectively sets how
        many objects from `lclist` will be processed in parallel.

    liststartindex : int or None
        This sets the index from where to start in `lclist`.

    listmaxobjects : int or None
        This sets the maximum number of objects in `lclist` to run
        period-finding for in this invocation. Together with `liststartindex`,
        `listmaxobjects` can be used to distribute processing over several
        independent machines if the number of light curves is very large.

    minobservations : int
        The minimum number of finite LC points required to process a light
        curve.

    excludeprocessed : bool
        If this is True, light curves that have existing period-finding result
        pickles in `outdir` will not be processed.

        FIXME: currently, this uses a dumb method of excluding already-processed
        files. A smarter way to do this is to (i) generate a SHA512 cachekey
        based on a repr of `{'lcfile', 'timecols', 'magcols', 'errcols',
        'lcformat', 'pfmethods', 'sigclip', 'getblssnr', 'pfkwargs'}`, (ii) make
        sure all list kwargs in the dict are sorted, (iii) check if the output
        file has the same cachekey in its filename (last 8 chars of cachekey
        should work), so the result was processed in exactly the same way as
        specifed in the input to this function, and can therefore be
        ignored. Will implement this later.

    Returns
    -------

    list of str
        A list of the period-finding pickles created for all of input LCs
        processed.

    '''

    # make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if (liststartindex is not None) and (listmaxobjects is None):
        lclist = lclist[liststartindex:]

    elif (liststartindex is None) and (listmaxobjects is not None):
        lclist = lclist[:listmaxobjects]

    elif (liststartindex is not None) and (listmaxobjects is not None):
        lclist = lclist[liststartindex:liststartindex+listmaxobjects]

    tasklist = [(x, outdir, timecols, magcols, errcols, lcformat, lcformatdir,
                 pfmethods, pfkwargs, getblssnr, sigclip, nperiodworkers,
                 minobservations,
                 excludeprocessed)
                for x in lclist]

    with ProcessPoolExecutor(max_workers=ncontrolworkers) as executor:
        resultfutures = executor.map(_runpf_worker, tasklist)

    results = [x for x in resultfutures]
    return results

This drives the overall parallel period processing for a list of LCs .


def parallel_pf_lcdir(lcdir,
                      outdir,
                      fileglob=None,
                      recursive=True,
                      timecols=None,
                      magcols=None,
                      errcols=None,
                      lcformat='hat-sql',
                      lcformatdir=None,
                      pfmethods=('gls','pdm','mav','win'),
                      pfkwargs=({},{},{},{}),
                      sigclip=10.0,
                      getblssnr=False,
                      nperiodworkers=NCPUS,
                      ncontrolworkers=1,
                      liststartindex=None,
                      listmaxobjects=None,
                      minobservations=500,
                      excludeprocessed=True):
    '''This runs parallel light curve period finding for directory of LCs.

    Parameters
    ----------

    lcdir : str
        The directory containing the LCs to process.

    outdir : str
        The directory where the resulting period-finding pickles will go.

    fileglob : str or None
        The UNIX file glob to use to search for LCs in `lcdir`. If None, the
        default file glob associated with the registered LC format will be used
        instead.

    recursive : bool
        If True, will search recursively in `lcdir` for light curves to process.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    pfmethods : list of str
        This is a list of period finding methods to run. Each element is a
        string matching the keys of the `PFMETHODS` dict above. By default, this
        runs GLS, PDM, AoVMH, and the spectral window Lomb-Scargle periodogram.

    pfkwargs : list of dicts
        This is used to provide any special kwargs as dicts to each
        period-finding method function specified in `pfmethods`.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    getblssnr : bool
        If this is True and BLS is one of the methods specified in `pfmethods`,
        will also calculate the stats for each best period in the BLS results:
        transit depth, duration, ingress duration, refit period and epoch, and
        the SNR of the transit.

    nperiodworkers : int
        The number of parallel period-finding workers to launch per object task.

    ncontrolworkers : int
        The number of controlling processes to launch. This effectively sets how
        many objects from `lclist` will be processed in parallel.

    liststartindex : int or None
        This sets the index from where to start in `lclist`.

    listmaxobjects : int or None
        This sets the maximum number of objects in `lclist` to run
        period-finding for in this invocation. Together with `liststartindex`,
        `listmaxobjects` can be used to distribute processing over several
        independent machines if the number of light curves is very large.

    minobservations : int
        The minimum number of finite LC points required to process a light
        curve.

    excludeprocessed : bool
        If this is True, light curves that have existing period-finding result
        pickles in `outdir` will not be processed.

        FIXME: currently, this uses a dumb method of excluding already-processed
        files. A smarter way to do this is to (i) generate a SHA512 cachekey
        based on a repr of `{'lcfile', 'timecols', 'magcols', 'errcols',
        'lcformat', 'pfmethods', 'sigclip', 'getblssnr', 'pfkwargs'}`, (ii) make
        sure all list kwargs in the dict are sorted, (iii) check if the output
        file has the same cachekey in its filename (last 8 chars of cachekey
        should work), so the result was processed in exactly the same way as
        specifed in the input to this function, and can therefore be
        ignored. Will implement this later.

    Returns
    -------

    list of str
        A list of the period-finding pickles created for all of input LCs
        processed.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    if not fileglob:
        fileglob = dfileglob

    # now find the files
    LOGINFO('searching for %s light curves in %s ...' % (lcformat, lcdir))

    if recursive is False:
        matching = glob.glob(os.path.join(lcdir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcdir,
                                              '**',
                                              fileglob),recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcdir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        # this helps us process things in deterministic order when we distribute
        # processing over several machines
        matching = sorted(matching)

        LOGINFO('found %s light curves, running pf...' % len(matching))

        return parallel_pf(matching,
                           outdir,
                           timecols=timecols,
                           magcols=magcols,
                           errcols=errcols,
                           lcformat=lcformat,
                           lcformatdir=lcformatdir,
                           pfmethods=pfmethods,
                           pfkwargs=pfkwargs,
                           getblssnr=getblssnr,
                           sigclip=sigclip,
                           nperiodworkers=nperiodworkers,
                           ncontrolworkers=ncontrolworkers,
                           liststartindex=liststartindex,
                           listmaxobjects=listmaxobjects,
                           minobservations=minobservations,
                           excludeprocessed=excludeprocessed)

    else:

        LOGERROR('no light curve files in %s format found in %s' % (lcformat,
                                                                    lcdir))
        return None

This runs parallel light curve period finding for directory of LCs .


def collect_nonperiodic_features(
        featuresdir,
        magcol,
        outfile,
        pklglob='varfeatures-*.pkl',
        featurestouse=NONPERIODIC_FEATURES_TO_COLLECT,
        maxobjects=None,
        labeldict=None,
        labeltype='binary',
):
    '''This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.

    '''

    # list of input pickles generated by varfeatures in lcproc.py
    pklist = glob.glob(os.path.join(featuresdir, pklglob))

    if maxobjects:
        pklist = pklist[:maxobjects]

    # fancy progress bar with tqdm if present
    if TQDM:
        listiterator = tqdm(pklist)
    else:
        listiterator = pklist

    # go through all the varfeatures arrays

    feature_dict = {'objectids':[],'magcol':magcol, 'availablefeatures':[]}

    LOGINFO('collecting features for magcol: %s' % magcol)

    for pkl in listiterator:

        with open(pkl,'rb') as infd:
            varf = pickle.load(infd)

        # update the objectid list
        objectid = varf['objectid']
        if objectid not in feature_dict['objectids']:
            feature_dict['objectids'].append(objectid)

        thisfeatures = varf[magcol]

        if featurestouse and len(featurestouse) > 0:
            featurestoget = featurestouse
        else:
            featurestoget = NONPERIODIC_FEATURES_TO_COLLECT

        # collect all the features for this magcol/objectid combination
        for feature in featurestoget:

            # update the global feature list if necessary
            if ((feature not in feature_dict['availablefeatures']) and
                (feature in thisfeatures)):

                feature_dict['availablefeatures'].append(feature)
                feature_dict[feature] = []

            if feature in thisfeatures:

                feature_dict[feature].append(
                    thisfeatures[feature]
                )

    # now that we've collected all the objects and their features, turn the list
    # into arrays, and then concatenate them
    for feat in feature_dict['availablefeatures']:
        feature_dict[feat] = np.array(feature_dict[feat])

    feature_dict['objectids'] = np.array(feature_dict['objectids'])

    feature_array = np.column_stack([feature_dict[feat] for feat in
                                     feature_dict['availablefeatures']])
    feature_dict['features_array'] = feature_array

    # if there's a labeldict available, use it to generate a label array. this
    # feature collection is now a training set.
    if isinstance(labeldict, dict):

        labelarray = np.zeros(feature_dict['objectids'].size, dtype=np.int64)

        # populate the labels for each object in the training set
        for ind, objectid in enumerate(feature_dict['objectids']):

            if objectid in labeldict:

                # if this is a binary classifier training set, convert bools to
                # ones and zeros
                if labeltype == 'binary':

                    if labeldict[objectid]:
                        labelarray[ind] = 1

                # otherwise, use the actual class label integer
                elif labeltype == 'classes':
                    labelarray[ind] = labeldict[objectid]

        feature_dict['labels_array'] = labelarray

    feature_dict['kwargs'] = {'pklglob':pklglob,
                              'featurestouse':featurestouse,
                              'maxobjects':maxobjects,
                              'labeltype':labeltype}

    # write the info to the output pickle
    with open(outfile,'wb') as outfd:
        pickle.dump(feature_dict, outfd, pickle.HIGHEST_PROTOCOL)

    # return the feature_dict
    return feature_dict

This collects variability features into arrays for use with the classifer .


def train_rf_classifier(
        collected_features,
        test_fraction=0.25,
        n_crossval_iterations=20,
        n_kfolds=5,
        crossval_scoring_metric='f1',
        classifier_to_pickle=None,
        nworkers=-1,
):

    '''This gets the best RF classifier after running cross-validation.

    - splits the training set into test/train samples
    - does `KFold` stratified cross-validation using `RandomizedSearchCV`
    - gets the `RandomForestClassifier` with the best performance after CV
    - gets the confusion matrix for the test set

    Runs on the output dict from functions that produce dicts similar to that
    produced by `collect_nonperiodic_features` above.

    Parameters
    ----------

    collected_features : dict or str
        This is either the dict produced by a `collect_*_features` function or
        the pickle produced by the same.

    test_fraction : float
        This sets the fraction of the input set that will be used as the
        test set after training.

    n_crossval_iterations : int
        This sets the number of iterations to use when running the
        cross-validation.

    n_kfolds : int
        This sets the number of K-folds to use on the data when doing a
        test-train split.

    crossval_scoring_metric : str
        This is a string that describes how the cross-validation score is
        calculated for each iteration. See the URL below for how to specify this
        parameter:

        http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter
        By default, this is tuned for binary classification and uses the F1
        scoring metric. Change the `crossval_scoring_metric` to another metric
        (probably 'accuracy') for multi-class classification, e.g. for periodic
        variable classification.

    classifier_to_pickle : str
        If this is a string indicating the name of a pickle file to write, will
        write the trained classifier to the pickle that can be later loaded and
        used to classify data.

    nworkers : int
        This is the number of parallel workers to use in the
        RandomForestClassifier. Set to -1 to use all CPUs on your machine.

    Returns
    -------

    dict
        A dict containing the trained classifier, cross-validation results, the
        input data set, and all input kwargs used is returned, along with
        cross-validation score metrics.

    '''

    if (isinstance(collected_features,str) and
        os.path.exists(collected_features)):
        with open(collected_features,'rb') as infd:
            fdict = pickle.load(infd)
    elif isinstance(collected_features, dict):
        fdict = collected_features
    else:
        LOGERROR("can't figure out the input collected_features arg")
        return None

    tfeatures = fdict['features_array']
    tlabels = fdict['labels_array']
    tfeaturenames = fdict['availablefeatures']
    tmagcol = fdict['magcol']
    tobjectids = fdict['objectids']

    # split the training set into training/test samples using stratification
    # to keep the same fraction of variable/nonvariables in each
    training_features, testing_features, training_labels, testing_labels = (
        train_test_split(
            tfeatures,
            tlabels,
            test_size=test_fraction,
            random_state=RANDSEED,
            stratify=tlabels
        )
    )

    # get a random forest classifier
    clf = RandomForestClassifier(n_jobs=nworkers,
                                 random_state=RANDSEED)

    # this is the grid def for hyperparam optimization
    rf_hyperparams = {
        "max_depth": [3,4,5,None],
        "n_estimators":sp_randint(100,2000),
        "max_features": sp_randint(1, 5),
        "min_samples_split": sp_randint(2, 11),
        "min_samples_leaf": sp_randint(2, 11),
    }

    # run the stratified kfold cross-validation on training features using our
    # random forest classifier object
    cvsearch = RandomizedSearchCV(
        clf,
        param_distributions=rf_hyperparams,
        n_iter=n_crossval_iterations,
        scoring=crossval_scoring_metric,
        cv=StratifiedKFold(n_splits=n_kfolds,
                           shuffle=True,
                           random_state=RANDSEED),
        random_state=RANDSEED
    )

    LOGINFO('running grid-search CV to optimize RF hyperparameters...')
    cvsearch_classifiers = cvsearch.fit(training_features,
                                        training_labels)

    # report on the classifiers' performance
    _gridsearch_report(cvsearch_classifiers.cv_results_)

    # get the best classifier after CV is done
    bestclf = cvsearch_classifiers.best_estimator_
    bestclf_score = cvsearch_classifiers.best_score_
    bestclf_hyperparams = cvsearch_classifiers.best_params_

    # test this classifier on the testing set
    test_predicted_labels = bestclf.predict(testing_features)

    recscore = recall_score(testing_labels, test_predicted_labels)
    precscore = precision_score(testing_labels,test_predicted_labels)
    f1score = f1_score(testing_labels, test_predicted_labels)
    confmatrix = confusion_matrix(testing_labels, test_predicted_labels)

    # write the classifier, its training/testing set, and its stats to the
    # pickle if requested
    outdict = {'features':tfeatures,
               'labels':tlabels,
               'feature_names':tfeaturenames,
               'magcol':tmagcol,
               'objectids':tobjectids,
               'kwargs':{'test_fraction':test_fraction,
                         'n_crossval_iterations':n_crossval_iterations,
                         'n_kfolds':n_kfolds,
                         'crossval_scoring_metric':crossval_scoring_metric,
                         'nworkers':nworkers},
               'collect_kwargs':fdict['kwargs'],
               'testing_features':testing_features,
               'testing_labels':testing_labels,
               'training_features':training_features,
               'training_labels':training_labels,
               'best_classifier':bestclf,
               'best_score':bestclf_score,
               'best_hyperparams':bestclf_hyperparams,
               'best_recall':recscore,
               'best_precision':precscore,
               'best_f1':f1score,
               'best_confmatrix':confmatrix}

    if classifier_to_pickle:

        with open(classifier_to_pickle,'wb') as outfd:
            pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

    # return this classifier and accompanying info
    return outdict

This gets the best RF classifier after running cross - validation .


def apply_rf_classifier(classifier,
                        varfeaturesdir,
                        outpickle,
                        maxobjects=None):
    '''This applys an RF classifier trained using `train_rf_classifier`
    to varfeatures pickles in `varfeaturesdir`.

    Parameters
    ----------

    classifier : dict or str
        This is the output dict or pickle created by `get_rf_classifier`. This
        will contain a `features_name` key that will be used to collect the same
        features used to train the classifier from the varfeatures pickles in
        varfeaturesdir.

    varfeaturesdir : str
        The directory containing the varfeatures pickles for objects that will
        be classified by the trained `classifier`.

    outpickle : str
        This is a filename for the pickle that will be written containing the
        result dict from this function.

    maxobjects : int
        This sets the number of objects to process in `varfeaturesdir`.

    Returns
    -------

    dict
        The classification results after running the trained `classifier` as
        returned as a dict. This contains predicted labels and their prediction
        probabilities.

    '''

    if isinstance(classifier,str) and os.path.exists(classifier):
        with open(classifier,'rb') as infd:
            clfdict = pickle.load(infd)
    elif isinstance(classifier, dict):
        clfdict = classifier
    else:
        LOGERROR("can't figure out the input classifier arg")
        return None

    # get the features to extract from clfdict
    if 'feature_names' not in clfdict:
        LOGERROR("feature_names not present in classifier input, "
                 "can't figure out which ones to extract from "
                 "varfeature pickles in %s" % varfeaturesdir)
        return None

    # get the feature labeltype, pklglob, and maxobjects from classifier's
    # collect_kwargs elem.
    featurestouse = clfdict['feature_names']
    pklglob = clfdict['collect_kwargs']['pklglob']
    magcol = clfdict['magcol']

    # extract the features used by the classifier from the varfeatures pickles
    # in varfeaturesdir using the pklglob provided
    featfile = os.path.join(
        os.path.dirname(outpickle),
        'actual-collected-features.pkl'
    )

    features = collect_nonperiodic_features(
        varfeaturesdir,
        magcol,
        featfile,
        pklglob=pklglob,
        featurestouse=featurestouse,
        maxobjects=maxobjects
    )

    # now use the trained classifier on these features
    bestclf = clfdict['best_classifier']

    predicted_labels = bestclf.predict(features['features_array'])

    # FIXME: do we need to use the probability calibration curves to fix these
    # probabilities? probably. figure out how to do this.
    predicted_label_probs = bestclf.predict_proba(
        features['features_array']
    )

    outdict = {
        'features':features,
        'featfile':featfile,
        'classifier':clfdict,
        'predicted_labels':predicted_labels,
        'predicted_label_probs':predicted_label_probs,
    }

    with open(outpickle,'wb') as outfd:
        pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

    return outdict

This applys an RF classifier trained using train_rf_classifier to varfeatures pickles in varfeaturesdir .


def plot_training_results(classifier,
                          classlabels,
                          outfile):
    '''This plots the training results from the classifier run on the training
    set.

    - plots the confusion matrix

    - plots the feature importances

    - FIXME: plot the learning curves too, see:
      http://scikit-learn.org/stable/modules/learning_curve.html

    Parameters
    ----------

    classifier : dict or str
        This is the output dict or pickle created by `get_rf_classifier`
        containing the trained classifier.

    classlabels : list of str
        This contains all of the class labels for the current classification
        problem.

    outfile : str
        This is the filename where the plots will be written.

    Returns
    -------

    str
        The path to the generated plot file.

    '''

    if isinstance(classifier,str) and os.path.exists(classifier):
        with open(classifier,'rb') as infd:
            clfdict = pickle.load(infd)
    elif isinstance(classifier, dict):
        clfdict = classifier
    else:
        LOGERROR("can't figure out the input classifier arg")
        return None

    confmatrix = clfdict['best_confmatrix']
    overall_feature_importances = clfdict[
        'best_classifier'
    ].feature_importances_
    feature_importances_per_tree = np.array([
        tree.feature_importances_
        for tree in clfdict['best_classifier'].estimators_
    ])
    stdev_feature_importances = np.std(feature_importances_per_tree,axis=0)

    feature_names = np.array(clfdict['feature_names'])

    plt.figure(figsize=(6.4*3.0,4.8))

    # confusion matrix
    plt.subplot(121)
    classes = np.array(classlabels)
    plt.imshow(confmatrix, interpolation='nearest', cmap=plt.cm.Blues)
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)
    plt.title('evaluation set confusion matrix')
    plt.ylabel('predicted class')
    plt.xlabel('actual class')

    thresh = confmatrix.max() / 2.
    for i, j in itertools.product(range(confmatrix.shape[0]),
                                  range(confmatrix.shape[1])):
        plt.text(j, i, confmatrix[i, j],
                 horizontalalignment="center",
                 color="white" if confmatrix[i, j] > thresh else "black")

    # feature importances
    plt.subplot(122)

    features = np.array(feature_names)
    sorted_ind = np.argsort(overall_feature_importances)[::-1]

    features = features[sorted_ind]
    feature_names = feature_names[sorted_ind]
    overall_feature_importances = overall_feature_importances[sorted_ind]
    stdev_feature_importances = stdev_feature_importances[sorted_ind]

    plt.bar(np.arange(0,features.size),
            overall_feature_importances,
            yerr=stdev_feature_importances,
            width=0.8,
            color='grey')
    plt.xticks(np.arange(0,features.size),
               features,
               rotation=90)
    plt.yticks([0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])
    plt.xlim(-0.75, features.size - 1.0 + 0.75)
    plt.ylim(0.0,0.9)
    plt.ylabel('relative importance')
    plt.title('relative importance of features')

    plt.subplots_adjust(wspace=0.1)

    plt.savefig(outfile,
                bbox_inches='tight',
                dpi=100)
    plt.close('all')
    return outfile

This plots the training results from the classifier run on the training set .


def _fourier_func(fourierparams, phase, mags):
    '''This returns a summed Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    phase,mags : np.array
        The input phase and magnitude areas to use as the basis for the cosine
        series. The phases are used directly to generate the values of the
        function, while the mags array is used to generate the zeroth order
        amplitude coefficient.

    Returns
    -------

    np.array
        The Fourier cosine series function evaluated over `phase`.

    '''

    # figure out the order from the length of the Fourier param list
    order = int(len(fourierparams)/2)

    # get the amplitude and phase coefficients
    f_amp = fourierparams[:order]
    f_pha = fourierparams[order:]

    # calculate all the individual terms of the series
    f_orders = [f_amp[x]*npcos(2.0*pi_value*x*phase + f_pha[x])
                for x in range(order)]

    # this is the zeroth order coefficient - a constant equal to median mag
    total_f = npmedian(mags)

    # sum the series
    for fo in f_orders:
        total_f += fo

    return total_f

This returns a summed Fourier cosine series .


def _fourier_chisq(fourierparams,
                   phase,
                   mags,
                   errs):
    '''This is the chisq objective function to be minimized by `scipy.minimize`.

    The parameters are the same as `_fourier_func` above. `errs` is used to
    calculate the chisq value.

    '''

    f = _fourier_func(fourierparams, phase, mags)
    chisq = npsum(((mags - f)*(mags - f))/(errs*errs))

    return chisq

def _fourier_chisq ( fourierparams , phase , mags , errs ) : f = _fourier_func ( fourierparams , phase , mags ) chisq = npsum ( ( ( mags - f ) * ( mags - f ) ) / ( errs * errs ) ) return chisq

This is the chisq objective function to be minimized by scipy . minimize .


def _fourier_residual(fourierparams,
                      phase,
                      mags):
    '''
    This is the residual objective function to be minimized by `scipy.leastsq`.

    The parameters are the same as `_fourier_func` above.

    '''

    f = _fourier_func(fourierparams, phase, mags)
    residual = mags - f

    return residual

def _fourier_residual ( fourierparams , phase , mags ) : f = _fourier_func ( fourierparams , phase , mags ) residual = mags - f return residual

This is the residual objective function to be minimized by scipy . leastsq .


def skyview_stamp(ra, decl,
                  survey='DSS2 Red',
                  scaling='Linear',
                  flip=True,
                  convolvewith=None,
                  forcefetch=False,
                  cachedir='~/.astrobase/stamp-cache',
                  timeout=10.0,
                  retry_failed=False,
                  savewcsheader=True,
                  verbose=False):
    '''This downloads a DSS FITS stamp centered on the coordinates specified.

    This wraps the function :py:func:`astrobase.services.skyview.get_stamp`,
    which downloads Digitized Sky Survey stamps in FITS format from the NASA
    SkyView service:

    https://skyview.gsfc.nasa.gov/current/cgi/query.pl

    Also adds some useful operations on top of the FITS file returned.

    Parameters
    ----------

    ra,decl : float
        The center coordinates for the stamp in decimal degrees.

    survey : str
        The survey name to get the stamp from. This is one of the
        values in the 'SkyView Surveys' option boxes on the SkyView
        webpage. Currently, we've only tested using 'DSS2 Red' as the value for
        this kwarg, but the other ones should work in principle.

    scaling : str
        This is the pixel value scaling function to use.

    flip : bool
        Will flip the downloaded image top to bottom. This should usually be
        True because matplotlib and FITS have different image coord origin
        conventions. Alternatively, set this to False and use the
        `origin='lower'` in any call to `matplotlib.pyplot.imshow` when plotting
        this image.

    convolvewith : astropy.convolution Kernel object or None
        If `convolvewith` is an astropy.convolution Kernel object from:

        http://docs.astropy.org/en/stable/convolution/kernels.html

        then, this function will return the stamp convolved with that
        kernel. This can be useful to see effects of wide-field telescopes (like
        the HATNet and HATSouth lenses) degrading the nominal 1 arcsec/px of
        DSS, causing blending of targets and any variability.

    forcefetch : bool
        If True, will disregard any existing cached copies of the stamp already
        downloaded corresponding to the requested center coordinates and
        redownload the FITS from the SkyView service.

    cachedir : str
        This is the path to the astrobase cache directory. All downloaded FITS
        stamps are stored here as .fits.gz files so we can immediately respond
        with the cached copy when a request is made for a coordinate center
        that's already been downloaded.

    timeout : float
        Sets the timeout in seconds to wait for a response from the NASA SkyView
        service.

    retry_failed : bool
        If the initial request to SkyView fails, and this is True, will retry
        until it succeeds.

    savewcsheader : bool
        If this is True, also returns the WCS header of the downloaded FITS
        stamp in addition to the FITS image itself. Useful for projecting object
        coordinates onto image xy coordinates for visualization.

    verbose : bool
        If True, indicates progress.

    Returns
    -------

    tuple or array or None
        This returns based on the value of `savewcsheader`:

        - If `savewcsheader=True`, returns a tuple:
          (FITS stamp image as a numpy array, FITS header)
        - If `savewcsheader=False`, returns only the FITS stamp image as numpy
          array.
        - If the stamp retrieval fails, returns None.

    '''

    stampdict = get_stamp(ra, decl,
                          survey=survey,
                          scaling=scaling,
                          forcefetch=forcefetch,
                          cachedir=cachedir,
                          timeout=timeout,
                          retry_failed=retry_failed,
                          verbose=verbose)
    #
    # DONE WITH FETCHING STUFF
    #
    if stampdict:

        # open the frame
        stampfits = pyfits.open(stampdict['fitsfile'])
        header = stampfits[0].header
        frame = stampfits[0].data
        stampfits.close()

        # finally, we can process the frame
        if flip:
            frame = np.flipud(frame)

        if verbose:
            LOGINFO('fetched stamp successfully for (%.3f, %.3f)'
                    % (ra, decl))

        if convolvewith:

            convolved = aconv.convolve(frame, convolvewith)
            if savewcsheader:
                return convolved, header
            else:
                return convolved

        else:

            if savewcsheader:
                return frame, header
            else:
                return frame

    else:
        LOGERROR('could not fetch the requested stamp for '
                 'coords: (%.3f, %.3f) from survey: %s and scaling: %s'
                 % (ra, decl, survey, scaling))
        return None

This downloads a DSS FITS stamp centered on the coordinates specified .


def plot_periodbase_lsp(lspinfo, outfile=None, plotdpi=100):

    '''Makes a plot of periodograms obtained from `periodbase` functions.

    This takes the output dict produced by any `astrobase.periodbase`
    period-finder function or a pickle filename containing such a dict and makes
    a periodogram plot.

    Parameters
    ----------

    lspinfo : dict or str
        If lspinfo is a dict, it must be a dict produced by an
        `astrobase.periodbase` period-finder function or a dict from your own
        period-finder function or routine that is of the form below with at
        least these keys::

            {'periods': np.array of all periods searched by the period-finder,
             'lspvals': np.array of periodogram power value for each period,
             'bestperiod': a float value that is the period with the highest
                           peak in the periodogram, i.e. the most-likely actual
                           period,
             'method': a three-letter code naming the period-finder used; must
                       be one of the keys in the `METHODLABELS` dict above,
             'nbestperiods': a list of the periods corresponding to periodogram
                             peaks (`nbestlspvals` below) to annotate on the
                             periodogram plot so they can be called out
                             visually,
             'nbestlspvals': a list of the power values associated with
                             periodogram peaks to annotate on the periodogram
                             plot so they can be called out visually; should be
                             the same length as `nbestperiods` above}

        If lspinfo is a str, then it must be a path to a pickle file that
        contains a dict of the form described above.

    outfile : str or None
        If this is a str, will write the periodogram plot to the file specified
        by this string. If this is None, will write to a file called
        'lsp-plot.png' in the current working directory.

    plotdpi : int
        Sets the resolution in DPI of the output periodogram plot PNG file.

    Returns
    -------

    str
        Absolute path to the periodogram plot file created.

    '''

    # get the lspinfo from a pickle file transparently
    if isinstance(lspinfo,str) and os.path.exists(lspinfo):
        LOGINFO('loading LSP info from pickle %s' % lspinfo)
        with open(lspinfo,'rb') as infd:
            lspinfo = pickle.load(infd)

    try:

        # get the things to plot out of the data
        periods = lspinfo['periods']
        lspvals = lspinfo['lspvals']
        bestperiod = lspinfo['bestperiod']
        lspmethod = lspinfo['method']

        # make the LSP plot on the first subplot
        plt.plot(periods, lspvals)
        plt.xscale('log',basex=10)
        plt.xlabel('Period [days]')
        plt.ylabel(PLOTYLABELS[lspmethod])
        plottitle = '%s best period: %.6f d' % (METHODSHORTLABELS[lspmethod],
                                                bestperiod)
        plt.title(plottitle)

        # show the best five peaks on the plot
        for bestperiod, bestpeak in zip(lspinfo['nbestperiods'],
                                        lspinfo['nbestlspvals']):

            plt.annotate('%.6f' % bestperiod,
                         xy=(bestperiod, bestpeak), xycoords='data',
                         xytext=(0.0,25.0), textcoords='offset points',
                         arrowprops=dict(arrowstyle="->"),fontsize='x-small')

        # make a grid
        plt.grid(color='#a9a9a9',
                 alpha=0.9,
                 zorder=0,
                 linewidth=1.0,
                 linestyle=':')

        # make the figure
        if outfile and isinstance(outfile, str):

            if outfile.endswith('.png'):
                plt.savefig(outfile,bbox_inches='tight',dpi=plotdpi)
            else:
                plt.savefig(outfile,bbox_inches='tight')

            plt.close()
            return os.path.abspath(outfile)

        elif dispok:

            plt.show()
            plt.close()
            return

        else:

            LOGWARNING('no output file specified and no $DISPLAY set, '
                       'saving to lsp-plot.png in current directory')
            outfile = 'lsp-plot.png'
            plt.savefig(outfile,bbox_inches='tight',dpi=plotdpi)
            plt.close()
            return os.path.abspath(outfile)

    except Exception as e:

        LOGEXCEPTION('could not plot this LSP, appears to be empty')
        return

Makes a plot of periodograms obtained from periodbase functions .


def lcdict_to_pickle(lcdict, outfile=None):
    '''This just writes the lcdict to a pickle.

    If outfile is None, then will try to get the name from the
    lcdict['objectid'] and write to <objectid>-hptxtlc.pkl. If that fails, will
    write to a file named hptxtlc.pkl'.

    '''

    if not outfile and lcdict['objectid']:
        outfile = '%s-hplc.pkl' % lcdict['objectid']
    elif not outfile and not lcdict['objectid']:
        outfile = 'hplc.pkl'

    with open(outfile,'wb') as outfd:
        pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    if os.path.exists(outfile):
        LOGINFO('lcdict for object: %s -> %s OK' % (lcdict['objectid'],
                                                    outfile))
        return outfile
    else:
        LOGERROR('could not make a pickle for this lcdict!')
        return None

This just writes the lcdict to a pickle .


def read_hatpi_pklc(lcfile):
    '''
    This just reads a pickle LC. Returns an lcdict.

    '''

    try:

        if lcfile.endswith('.gz'):
            infd = gzip.open(lcfile,'rb')
        else:
            infd = open(lcfile,'rb')

        lcdict = pickle.load(infd)
        infd.close()

        return lcdict

    except UnicodeDecodeError:

        if lcfile.endswith('.gz'):
            infd = gzip.open(lcfile,'rb')
        else:
            infd = open(lcfile,'rb')

        LOGWARNING('pickle %s was probably from Python 2 '
                   'and failed to load without using "latin1" encoding. '
                   'This is probably a numpy issue: '
                   'http://stackoverflow.com/q/11305790' % lcfile)
        lcdict = pickle.load(infd, encoding='latin1')
        infd.close()

        return lcdict

This just reads a pickle LC . Returns an lcdict .


def concatenate_textlcs(lclist,
                        sortby='rjd',
                        normalize=True):
    '''This concatenates a list of light curves.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.

    '''

    # read the first light curve
    lcdict = read_hatpi_textlc(lclist[0])

    # track which LC goes where
    # initial LC
    lccounter = 0
    lcdict['concatenated'] = {lccounter: os.path.abspath(lclist[0])}
    lcdict['lcn'] = np.full_like(lcdict['rjd'], lccounter)

    # normalize if needed
    if normalize:

        for col in MAGCOLS:

            if col in lcdict:
                thismedval = np.nanmedian(lcdict[col])

                # handle fluxes
                if col in ('ifl1','ifl2','ifl3'):
                    lcdict[col] = lcdict[col] / thismedval
                # handle mags
                else:
                    lcdict[col] = lcdict[col] - thismedval

    # now read the rest
    for lcf in lclist[1:]:

        thislcd = read_hatpi_textlc(lcf)

        # if the columns don't agree, skip this LC
        if thislcd['columns'] != lcdict['columns']:
            LOGERROR('file %s does not have the '
                     'same columns as first file %s, skipping...'
                     % (lcf, lclist[0]))
            continue

        # otherwise, go ahead and start concatenatin'
        else:

            LOGINFO('adding %s (ndet: %s) to %s (ndet: %s)'
                    % (lcf,
                       thislcd['objectinfo']['ndet'],
                       lclist[0],
                       lcdict[lcdict['columns'][0]].size))

            # update LC tracking
            lccounter = lccounter + 1
            lcdict['concatenated'][lccounter] = os.path.abspath(lcf)
            lcdict['lcn'] = np.concatenate((
                lcdict['lcn'],
                np.full_like(thislcd['rjd'],lccounter)
            ))

            # concatenate the columns
            for col in lcdict['columns']:

                # handle normalization for magnitude columns
                if normalize and col in MAGCOLS:

                    thismedval = np.nanmedian(thislcd[col])

                    # handle fluxes
                    if col in ('ifl1','ifl2','ifl3'):
                        thislcd[col] = thislcd[col] / thismedval
                    # handle mags
                    else:
                        thislcd[col] = thislcd[col] - thismedval

                # concatenate the values
                lcdict[col] = np.concatenate((lcdict[col], thislcd[col]))

    #
    # now we're all done concatenatin'
    #

    # make sure to add up the ndet
    lcdict['objectinfo']['ndet'] = lcdict[lcdict['columns'][0]].size

    # update the stations
    lcdict['objectinfo']['stations'] = [
        'HP%s' % x for x in np.unique(lcdict['stf']).tolist()
    ]

    # update the total LC count
    lcdict['nconcatenated'] = lccounter + 1

    # if we're supposed to sort by a column, do so
    if sortby and sortby in [x[0] for x in COLDEFS]:

        LOGINFO('sorting concatenated light curve by %s...' % sortby)
        sortind = np.argsort(lcdict[sortby])

        # sort all the measurement columns by this index
        for col in lcdict['columns']:
            lcdict[col] = lcdict[col][sortind]

        # make sure to sort the lcn index as well
        lcdict['lcn'] = lcdict['lcn'][sortind]

    LOGINFO('done. concatenated light curve has %s detections' %
            lcdict['objectinfo']['ndet'])
    return lcdict

This concatenates a list of light curves .


def concatenate_textlcs_for_objectid(lcbasedir,
                                     objectid,
                                     aperture='TF1',
                                     postfix='.gz',
                                     sortby='rjd',
                                     normalize=True,
                                     recursive=True):
    '''This concatenates all text LCs for an objectid with the given aperture.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    lcbasedir is the directory to start searching in.

    objectid is the object to search for.

    aperture is the aperture postfix to use: (TF1 = aperture 1,
                                              TF2 = aperture 2,
                                              TF3 = aperture 3)

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero, and the whole light curve is then normalized to the
    global median magnitude for each magnitude column.

    If recursive is True, then the function will search recursively in lcbasedir
    for any light curves matching the specified criteria. This may take a while,
    especially on network filesystems.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.

    '''
    LOGINFO('looking for light curves for %s, aperture %s in directory: %s'
            % (objectid, aperture, lcbasedir))

    if recursive is False:

        matching = glob.glob(os.path.join(lcbasedir,
                                          '*%s*%s*%s' % (objectid,
                                                         aperture,
                                                         postfix)))
    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcbasedir,
                                              '**',
                                              '*%s*%s*%s' % (objectid,
                                                             aperture,
                                                             postfix)),
                                 recursive=True)
            LOGINFO('found %s files: %s' % (len(matching), repr(matching)))

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcbasedir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              '*%s*%s*%s' % (objectid,
                                                             aperture,
                                                             postfix))
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)
                        LOGINFO(
                            'found %s in dir: %s' % (repr(foundfiles),
                                                     os.path.join(root,sdir))
                        )

    # now that we have all the files, concatenate them
    # a single file will be returned as normalized
    if matching and len(matching) > 0:
        clcdict = concatenate_textlcs(matching,
                                      sortby=sortby,
                                      normalize=normalize)
        return clcdict
    else:
        LOGERROR('did not find any light curves for %s and aperture %s' %
                 (objectid, aperture))
        return None

This concatenates all text LCs for an objectid with the given aperture .


def concat_write_pklc(lcbasedir,
                      objectid,
                      aperture='TF1',
                      postfix='.gz',
                      sortby='rjd',
                      normalize=True,
                      outdir=None,
                      recursive=True):
    '''This concatenates all text LCs for the given object and writes to a pklc.

    Basically a rollup for the concatenate_textlcs_for_objectid and
    lcdict_to_pickle functions.

    '''

    concatlcd = concatenate_textlcs_for_objectid(lcbasedir,
                                                 objectid,
                                                 aperture=aperture,
                                                 sortby=sortby,
                                                 normalize=normalize,
                                                 recursive=recursive)

    if not outdir:
        outdir = 'pklcs'

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    outfpath = os.path.join(outdir, '%s-%s-pklc.pkl' % (concatlcd['objectid'],
                                                        aperture))
    pklc = lcdict_to_pickle(concatlcd, outfile=outfpath)
    return pklc

def concat_write_pklc ( lcbasedir , objectid , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True ) : concatlcd = concatenate_textlcs_for_objectid ( lcbasedir , objectid , aperture = aperture , sortby = sortby , normalize = normalize , recursive = recursive ) if not outdir : outdir = 'pklcs' if not os . path . exists ( outdir ) : os . mkdir ( outdir ) outfpath = os . path . join ( outdir , '%s-%s-pklc.pkl' % ( concatlcd [ 'objectid' ] , aperture ) ) pklc = lcdict_to_pickle ( concatlcd , outfile = outfpath ) return pklc

This concatenates all text LCs for the given object and writes to a pklc .


def parallel_concat_worker(task):
    '''
    This is a worker for the function below.

    task[0] = lcbasedir
    task[1] = objectid
    task[2] = {'aperture','postfix','sortby','normalize','outdir','recursive'}

    '''

    lcbasedir, objectid, kwargs = task

    try:
        return concat_write_pklc(lcbasedir, objectid, **kwargs)
    except Exception as e:
        LOGEXCEPTION('failed LC concatenation for %s in %s'
                     % (objectid, lcbasedir))
        return None

def
parallel_concat_worker
(
task
)
:
lcbasedir
,
objectid
,
kwargs
=
task
try
:
return
concat_write_pklc
(
lcbasedir
,
objectid
,
*
*
kwargs
)
except
Exception
as
e
:
LOGEXCEPTION
(
'failed LC concatenation for %s in %s'
%
(
objectid
,
lcbasedir
)
)
return

This is a worker for the function below .


def parallel_concat_lcdir(lcbasedir,
                          objectidlist,
                          aperture='TF1',
                          postfix='.gz',
                          sortby='rjd',
                          normalize=True,
                          outdir=None,
                          recursive=True,
                          nworkers=32,
                          maxworkertasks=1000):
    '''This concatenates all text LCs for the given objectidlist.

    '''

    if not outdir:
        outdir = 'pklcs'

    if not os.path.exists(outdir):
        os.mkdir(outdir)

    tasks = [(lcbasedir, x, {'aperture':aperture,
                             'postfix':postfix,
                             'sortby':sortby,
                             'normalize':normalize,
                             'outdir':outdir,
                             'recursive':recursive}) for x in objectidlist]

    pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)
    results = pool.map(parallel_concat_worker, tasks)

    pool.close()
    pool.join()

    return {x:y for (x,y) in zip(objectidlist, results)}

def parallel_concat_lcdir ( lcbasedir , objectidlist , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True , nworkers = 32 , maxworkertasks = 1000 ) : if not outdir : outdir = 'pklcs' if not os . path . exists ( outdir ) : os . mkdir ( outdir ) tasks = [ ( lcbasedir , x , { 'aperture' : aperture , 'postfix' : postfix , 'sortby' : sortby , 'normalize' : normalize , 'outdir' : outdir , 'recursive' : recursive } ) for x in objectidlist ] pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks ) results = pool . map ( parallel_concat_worker , tasks ) pool . close ( ) pool . join ( ) return { x : y for ( x , y ) in zip ( objectidlist , results ) }

This concatenates all text LCs for the given objectidlist .


def merge_hatpi_textlc_apertures(lclist):
    '''This merges all TFA text LCs with separate apertures for a single object.

    The framekey column will be used as the join column across all light curves
    in lclist. Missing values will be filled in with nans. This function assumes
    all light curves are in the format specified in COLDEFS above and readable
    by read_hatpi_textlc above (i.e. have a single column for TFA mags for a
    specific aperture at the end).

    '''

    lcaps = {}
    framekeys = []

    for lc in lclist:

        lcd = read_hatpi_textlc(lc)

        # figure what aperture this is and put it into the lcdict. if two LCs
        # with the same aperture (i.e. TF1 and TF1) are provided, the later one
        # in the lclist will overwrite the previous one,
        for col in lcd['columns']:
            if col.startswith('itf'):
                lcaps[col] = lcd
        thisframekeys = lcd['frk'].tolist()
        framekeys.extend(thisframekeys)

    # uniqify the framekeys
    framekeys = sorted(list(set(framekeys)))

This merges all TFA text LCs with separate apertures for a single object .


def generate_hatpi_binnedlc_pkl(binnedpklf, textlcf, timebinsec,
                                outfile=None):
    '''
    This reads the binned LC and writes it out to a pickle.

    '''

    binlcdict = read_hatpi_binnedlc(binnedpklf, textlcf, timebinsec)

    if binlcdict:
        if outfile is None:
            outfile = os.path.join(
                os.path.dirname(binnedpklf),
                '%s-hplc.pkl' % (
                    os.path.basename(binnedpklf).replace('sec-lc.pkl.gz','')
                )
            )

        return lcdict_to_pickle(binlcdict, outfile=outfile)
    else:
        LOGERROR('could not read binned HATPI LC: %s' % binnedpklf)
        return None

def
generate_hatpi_binnedlc_pkl
(
binnedpklf
,
textlcf
,
timebinsec
,
outfile
=
)
:
binlcdict
=
read_hatpi_binnedlc
(
binnedpklf
,
textlcf
,
timebinsec
)
if
binlcdict
:
if
outfile
is
:
outfile
=
os
.
path
.
join
(
os
.
path
.
dirname
(
binnedpklf
)
,
'%s-hplc.pkl'
%
(
os
.
path
.
basename
(
binnedpklf
)
.
replace
(
'sec-lc.pkl.gz'
,
''
)
)
)
return
lcdict_to_pickle
(
binlcdict
,
outfile
=
outfile
)
else
:
LOGERROR
(
'could not read binned HATPI LC: %s'
%
binnedpklf
)
return

This reads the binned LC and writes it out to a pickle .


def parallel_gen_binnedlc_pkls(binnedpkldir,
                               textlcdir,
                               timebinsec,
                               binnedpklglob='*binned*sec*.pkl',
                               textlcglob='*.tfalc.TF1*'):
    '''
    This generates the binnedlc pkls for a directory of such files.

    FIXME: finish this

    '''

    binnedpkls = sorted(glob.glob(os.path.join(binnedpkldir, binnedpklglob)))

    # find all the textlcs associated with these
    textlcs = []

    for bpkl in binnedpkls:

        objectid = HATIDREGEX.findall(bpkl)
        if objectid is not None:
            objectid = objectid[0]

        searchpath = os.path.join(textlcdir, '%s-%s' % (objectid, textlcglob))
        textlcf = glob.glob(searchpath)
        if textlcf:
            textlcs.append(textlcf)
        else:
            textlcs.append(None)

This generates the binnedlc pkls for a directory of such files .