def read_csv_lightcurve(lcfile):
    '''
    This reads in a K2 lightcurve in CSV format. Transparently reads gzipped
    files.

    Parameters
    ----------

    lcfile : str
        The light curve file to read.

    Returns
    -------

    dict
        Returns an lcdict.

    '''

    # read in the file first
    if '.gz' in os.path.basename(lcfile):
        LOGINFO('reading gzipped K2 LC: %s' % lcfile)
        infd = gzip.open(lcfile,'rb')
    else:
        LOGINFO('reading K2 LC: %s' % lcfile)
        infd = open(lcfile,'rb')

    lctext = infd.read().decode()
    infd.close()

    # figure out the header and get the LC columns
    lcstart = lctext.index('# LIGHTCURVE\n')
    lcheader = lctext[:lcstart+12]
    lccolumns = lctext[lcstart+13:].split('\n')
    lccolumns = [x.split(',') for x in lccolumns if len(x) > 0]

    # initialize the lcdict and parse the CSV header
    lcdict = _parse_csv_header(lcheader)

    # tranpose the LC rows into columns
    lccolumns = list(zip(*lccolumns))

    # write the columns to the dict
    for colind, col in enumerate(lcdict['columns']):

        # this picks out the caster to use when reading each column using the
        # definitions in the lcutils.COLUMNDEFS dictionary
        lcdict[col.lower()] = np.array([COLUMNDEFS[col][2](x)
                                        for x in lccolumns[colind]])

    lcdict['columns'] = [x.lower() for x in lcdict['columns']]

    return lcdict

This reads in a K2 lightcurve in CSV format . Transparently reads gzipped files .


def _starfeatures_worker(task):
    '''
    This wraps starfeatures.

    '''

    try:
        (lcfile, outdir, kdtree, objlist,
         lcflist, neighbor_radius_arcsec,
         deredden, custom_bandpasses, lcformat, lcformatdir) = task

        return get_starfeatures(lcfile, outdir,
                                kdtree, objlist, lcflist,
                                neighbor_radius_arcsec,
                                deredden=deredden,
                                custom_bandpasses=custom_bandpasses,
                                lcformat=lcformat,
                                lcformatdir=lcformatdir)
    except Exception as e:
        return None

def _starfeatures_worker ( task ) : try : ( lcfile , outdir , kdtree , objlist , lcflist , neighbor_radius_arcsec , deredden , custom_bandpasses , lcformat , lcformatdir ) = task return get_starfeatures ( lcfile , outdir , kdtree , objlist , lcflist , neighbor_radius_arcsec , deredden = deredden , custom_bandpasses = custom_bandpasses , lcformat = lcformat , lcformatdir = lcformatdir ) except Exception as e : return None


def serial_starfeatures(lclist,
                        outdir,
                        lc_catalog_pickle,
                        neighbor_radius_arcsec,
                        maxobjects=None,
                        deredden=True,
                        custom_bandpasses=None,
                        lcformat='hat-sql',
                        lcformatdir=None):
    '''This drives the `get_starfeatures` function for a collection of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The output directory where the results will be placed.

    lc_catalog_pickle : str
        The path to a catalog containing at a dict with least:

        - an object ID array accessible with `dict['objects']['objectid']`

        - an LC filename array accessible with `dict['objects']['lcfname']`

        - a `scipy.spatial.KDTree` or `cKDTree` object to use for finding
          neighbors for each object accessible with `dict['kdtree']`

        A catalog pickle of the form needed can be produced using
        :py:func:`astrobase.lcproc.catalogs.make_lclist` or
        :py:func:`astrobase.lcproc.catalogs.filter_lclist`.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    maxobjects : int
        The number of objects to process from `lclist`.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        A list of all star features pickles produced.

    '''
    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        lclist = lclist[:maxobjects]

    # read in the kdtree pickle
    with open(lc_catalog_pickle, 'rb') as infd:
        kdt_dict = pickle.load(infd)

    kdt = kdt_dict['kdtree']
    objlist = kdt_dict['objects']['objectid']
    objlcfl = kdt_dict['objects']['lcfname']

    tasks = [(x, outdir, kdt, objlist, objlcfl,
              neighbor_radius_arcsec,
              deredden, custom_bandpasses,
              lcformat, lcformatdir) for x in lclist]

    for task in tqdm(tasks):
        result = _starfeatures_worker(task)

    return result

This drives the get_starfeatures function for a collection of LCs .


def parallel_starfeatures(lclist,
                          outdir,
                          lc_catalog_pickle,
                          neighbor_radius_arcsec,
                          maxobjects=None,
                          deredden=True,
                          custom_bandpasses=None,
                          lcformat='hat-sql',
                          lcformatdir=None,
                          nworkers=NCPUS):
    '''This runs `get_starfeatures` in parallel for all light curves in `lclist`.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The output directory where the results will be placed.

    lc_catalog_pickle : str
        The path to a catalog containing at a dict with least:

        - an object ID array accessible with `dict['objects']['objectid']`

        - an LC filename array accessible with `dict['objects']['lcfname']`

        - a `scipy.spatial.KDTree` or `cKDTree` object to use for finding
          neighbors for each object accessible with `dict['kdtree']`

        A catalog pickle of the form needed can be produced using
        :py:func:`astrobase.lcproc.catalogs.make_lclist` or
        :py:func:`astrobase.lcproc.catalogs.filter_lclist`.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    maxobjects : int
        The number of objects to process from `lclist`.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of the input light curve filename and the
        output star features pickle for each LC processed.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        lclist = lclist[:maxobjects]

    # read in the kdtree pickle
    with open(lc_catalog_pickle, 'rb') as infd:
        kdt_dict = pickle.load(infd)

    kdt = kdt_dict['kdtree']
    objlist = kdt_dict['objects']['objectid']
    objlcfl = kdt_dict['objects']['lcfname']

    tasks = [(x, outdir, kdt, objlist, objlcfl,
              neighbor_radius_arcsec,
              deredden, custom_bandpasses, lcformat) for x in lclist]

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(_starfeatures_worker, tasks)

    results = [x for x in resultfutures]
    resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}

    return resdict

This runs get_starfeatures in parallel for all light curves in lclist .


def parallel_starfeatures_lcdir(lcdir,
                                outdir,
                                lc_catalog_pickle,
                                neighbor_radius_arcsec,
                                fileglob=None,
                                maxobjects=None,
                                deredden=True,
                                custom_bandpasses=None,
                                lcformat='hat-sql',
                                lcformatdir=None,
                                nworkers=NCPUS,
                                recursive=True):
    '''This runs parallel star feature extraction for a directory of LCs.

    Parameters
    ----------

    lcdir : list of str
        The directory to search for light curves.

    outdir : str
        The output directory where the results will be placed.

    lc_catalog_pickle : str
        The path to a catalog containing at a dict with least:

        - an object ID array accessible with `dict['objects']['objectid']`

        - an LC filename array accessible with `dict['objects']['lcfname']`

        - a `scipy.spatial.KDTree` or `cKDTree` object to use for finding
          neighbors for each object accessible with `dict['kdtree']`

        A catalog pickle of the form needed can be produced using
        :py:func:`astrobase.lcproc.catalogs.make_lclist` or
        :py:func:`astrobase.lcproc.catalogs.filter_lclist`.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    fileglob : str
        The UNIX file glob to use to search for the light curves in `lcdir`. If
        None, the default value for the light curve format specified will be
        used.

    maxobjects : int
        The number of objects to process from `lclist`.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of the input light curve filename and the
        output star features pickle for each LC processed.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    if not fileglob:
        fileglob = dfileglob

    # now find the files
    LOGINFO('searching for %s light curves in %s ...' % (lcformat, lcdir))

    if recursive is False:
        matching = glob.glob(os.path.join(lcdir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(lcdir,
                                              '**',
                                              fileglob),recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(lcdir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        LOGINFO('found %s light curves, getting starfeatures...' %
                len(matching))

        return parallel_starfeatures(matching,
                                     outdir,
                                     lc_catalog_pickle,
                                     neighbor_radius_arcsec,
                                     deredden=deredden,
                                     custom_bandpasses=custom_bandpasses,
                                     maxobjects=maxobjects,
                                     lcformat=lcformat,
                                     lcformatdir=lcformatdir,
                                     nworkers=nworkers)

    else:

        LOGERROR('no light curve files in %s format found in %s' % (lcformat,
                                                                    lcdir))
        return None

This runs parallel star feature extraction for a directory of LCs .


def pwd_phasebin(phases, mags, binsize=0.002, minbin=9):
    '''
    This bins the phased mag series using the given binsize.

    '''

    bins = np.arange(0.0, 1.0, binsize)
    binnedphaseinds = npdigitize(phases, bins)

    binnedphases, binnedmags = [], []

    for x in npunique(binnedphaseinds):

        thisbin_inds = binnedphaseinds == x
        thisbin_phases = phases[thisbin_inds]
        thisbin_mags = mags[thisbin_inds]
        if thisbin_inds.size > minbin:
            binnedphases.append(npmedian(thisbin_phases))
            binnedmags.append(npmedian(thisbin_mags))

    return np.array(binnedphases), np.array(binnedmags)

def pwd_phasebin ( phases , mags , binsize = 0.002 , minbin = 9 ) : bins = np . arange ( 0.0 , 1.0 , binsize ) binnedphaseinds = npdigitize ( phases , bins ) binnedphases , binnedmags = [ ] , [ ] for x in npunique ( binnedphaseinds ) : thisbin_inds = binnedphaseinds == x thisbin_phases = phases [ thisbin_inds ] thisbin_mags = mags [ thisbin_inds ] if thisbin_inds . size > minbin : binnedphases . append ( npmedian ( thisbin_phases ) ) binnedmags . append ( npmedian ( thisbin_mags ) ) return np . array ( binnedphases ) , np . array ( binnedmags )

This bins the phased mag series using the given binsize .


def pdw_worker(task):
    '''
    This is the parallel worker for the function below.

    task[0] = frequency for this worker
    task[1] = times array
    task[2] = mags array
    task[3] = fold_time
    task[4] = j_range
    task[5] = keep_threshold_1
    task[6] = keep_threshold_2
    task[7] = phasebinsize

    we don't need errs for the worker.

    '''

    frequency = task[0]
    times, modmags = task[1], task[2]
    fold_time = task[3]
    j_range = range(task[4])
    keep_threshold_1 = task[5]
    keep_threshold_2 = task[6]
    phasebinsize = task[7]

    try:

        period = 1.0/frequency

        # use the common phaser to phase and sort the mag
        phased = phase_magseries(times,
                                 modmags,
                                 period,
                                 fold_time,
                                 wrap=False,
                                 sort=True)

        # bin in phase if requested, this turns this into a sort of PDM method
        if phasebinsize is not None and phasebinsize > 0:
            bphased = pwd_phasebin(phased['phase'],
                                   phased['mags'],
                                   binsize=phasebinsize)
            phase_sorted = bphased[0]
            mod_mag_sorted = bphased[1]
            j_range = range(len(mod_mag_sorted) - 1)
        else:
            phase_sorted = phased['phase']
            mod_mag_sorted = phased['mags']

        # now calculate the string length
        rolledmags = nproll(mod_mag_sorted,1)
        rolledphases = nproll(phase_sorted,1)
        strings = (
            (rolledmags - mod_mag_sorted)*(rolledmags - mod_mag_sorted) +
            (rolledphases - phase_sorted)*(rolledphases - phase_sorted)
        )
        strings[0] = (
            ((mod_mag_sorted[0] - mod_mag_sorted[-1]) *
             (mod_mag_sorted[0] - mod_mag_sorted[-1])) +
            ((phase_sorted[0] - phase_sorted[-1] + 1) *
             (phase_sorted[0] - phase_sorted[-1] + 1))
        )
        strlen = npsum(npsqrt(strings))

        if (keep_threshold_1 < strlen < keep_threshold_2):
            p_goodflag  = True
        else:
            p_goodflag = False

        return (period, strlen, p_goodflag)

    except Exception as e:

        LOGEXCEPTION('error in DWP')
        return(period, npnan, False)

This is the parallel worker for the function below .


def _periodicfeatures_worker(task):
    '''
    This is a parallel worker for the drivers below.

    '''

    pfpickle, lcbasedir, outdir, starfeatures, kwargs = task

    try:

        return get_periodicfeatures(pfpickle,
                                    lcbasedir,
                                    outdir,
                                    starfeatures=starfeatures,
                                    **kwargs)

    except Exception as e:

        LOGEXCEPTION('failed to get periodicfeatures for %s' % pfpickle)

def
_periodicfeatures_worker
(
task
)
:
pfpickle
,
lcbasedir
,
outdir
,
starfeatures
,
kwargs
=
task
try
:
return
get_periodicfeatures
(
pfpickle
,
lcbasedir
,
outdir
,
starfeatures
=
starfeatures
,
*
*
kwargs
)
except
Exception
as
e
:
LOGEXCEPTION
(
'failed to get periodicfeatures for %s'
%
pfpickle
)

This is a parallel worker for the drivers below .


def serial_periodicfeatures(pfpkl_list,
                            lcbasedir,
                            outdir,
                            starfeaturesdir=None,
                            fourierorder=5,
                            # these are depth, duration, ingress duration
                            transitparams=(-0.01,0.1,0.1),
                            # these are depth, duration, depth ratio, secphase
                            ebparams=(-0.2,0.3,0.7,0.5),
                            pdiff_threshold=1.0e-4,
                            sidereal_threshold=1.0e-4,
                            sampling_peak_multiplier=5.0,
                            sampling_startp=None,
                            sampling_endp=None,
                            starfeatures=None,
                            timecols=None,
                            magcols=None,
                            errcols=None,
                            lcformat='hat-sql',
                            lcformatdir=None,
                            sigclip=10.0,
                            verbose=False,
                            maxobjects=None):
    '''This drives the periodicfeatures collection for a list of periodfinding
    pickles.

    Parameters
    ----------

    pfpkl_list : list of str
        The list of period-finding pickles to use.

    lcbasedir : str
        The base directory where the associated light curves are located.

    outdir : str
        The directory where the results will be written.

    starfeaturesdir : str or None
        The directory containing the `starfeatures-<objectid>.pkl` files for
        each object to use calculate neighbor proximity light curve features.

    fourierorder : int
        The Fourier order to use to generate sinusoidal function and fit that to
        the phased light curve.

    transitparams : list of floats
        The transit depth, duration, and ingress duration to use to generate a
        trapezoid planet transit model fit to the phased light curve. The period
        used is the one provided in `period`, while the epoch is automatically
        obtained from a spline fit to the phased light curve.

    ebparams : list of floats
        The primary eclipse depth, eclipse duration, the primary-secondary depth
        ratio, and the phase of the secondary eclipse to use to generate an
        eclipsing binary model fit to the phased light curve. The period used is
        the one provided in `period`, while the epoch is automatically obtained
        from a spline fit to the phased light curve.

    pdiff_threshold : float
        This is the max difference between periods to consider them the same.

    sidereal_threshold : float
        This is the max difference between any of the 'best' periods and the
        sidereal day periods to consider them the same.

    sampling_peak_multiplier : float
        This is the minimum multiplicative factor of a 'best' period's
        normalized periodogram peak over the sampling periodogram peak at the
        same period required to accept the 'best' period as possibly real.

    sampling_startp, sampling_endp : float
        If the `pgramlist` doesn't have a time-sampling Lomb-Scargle
        periodogram, it will be obtained automatically. Use these kwargs to
        control the minimum and maximum period interval to be searched when
        generating this periodogram.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate progress while working.

    maxobjects : int
        The total number of objects to process from `pfpkl_list`.

    Returns
    -------

    Nothing.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (fileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        pfpkl_list = pfpkl_list[:maxobjects]

    LOGINFO('%s periodfinding pickles to process' % len(pfpkl_list))

    # if the starfeaturedir is provided, try to find a starfeatures pickle for
    # each periodfinding pickle in pfpkl_list
    if starfeaturesdir and os.path.exists(starfeaturesdir):

        starfeatures_list = []

        LOGINFO('collecting starfeatures pickles...')

        for pfpkl in pfpkl_list:

            sfpkl1 = os.path.basename(pfpkl).replace('periodfinding',
                                                     'starfeatures')
            sfpkl2 = sfpkl1.replace('.gz','')

            sfpath1 = os.path.join(starfeaturesdir, sfpkl1)
            sfpath2 = os.path.join(starfeaturesdir, sfpkl2)

            if os.path.exists(sfpath1):
                starfeatures_list.append(sfpkl1)
            elif os.path.exists(sfpath2):
                starfeatures_list.append(sfpkl2)
            else:
                starfeatures_list.append(None)

    else:

        starfeatures_list = [None for x in pfpkl_list]

    # generate the task list
    kwargs = {'fourierorder':fourierorder,
              'transitparams':transitparams,
              'ebparams':ebparams,
              'pdiff_threshold':pdiff_threshold,
              'sidereal_threshold':sidereal_threshold,
              'sampling_peak_multiplier':sampling_peak_multiplier,
              'sampling_startp':sampling_startp,
              'sampling_endp':sampling_endp,
              'timecols':timecols,
              'magcols':magcols,
              'errcols':errcols,
              'lcformat':lcformat,
              'lcformatdir':lcformatdir,
              'sigclip':sigclip,
              'verbose':verbose}

    tasks = [(x, lcbasedir, outdir, y, kwargs) for (x,y) in
             zip(pfpkl_list, starfeatures_list)]

    LOGINFO('processing periodfinding pickles...')

    for task in tqdm(tasks):
        _periodicfeatures_worker(task)

This drives the periodicfeatures collection for a list of periodfinding pickles .


def parallel_periodicfeatures(pfpkl_list,
                              lcbasedir,
                              outdir,
                              starfeaturesdir=None,
                              fourierorder=5,
                              # these are depth, duration, ingress duration
                              transitparams=(-0.01,0.1,0.1),
                              # these are depth, duration, depth ratio, secphase
                              ebparams=(-0.2,0.3,0.7,0.5),
                              pdiff_threshold=1.0e-4,
                              sidereal_threshold=1.0e-4,
                              sampling_peak_multiplier=5.0,
                              sampling_startp=None,
                              sampling_endp=None,
                              timecols=None,
                              magcols=None,
                              errcols=None,
                              lcformat='hat-sql',
                              lcformatdir=None,
                              sigclip=10.0,
                              verbose=False,
                              maxobjects=None,
                              nworkers=NCPUS):
    '''This runs periodic feature generation in parallel for all periodfinding
    pickles in the input list.

    Parameters
    ----------

    pfpkl_list : list of str
        The list of period-finding pickles to use.

    lcbasedir : str
        The base directory where the associated light curves are located.

    outdir : str
        The directory where the results will be written.

    starfeaturesdir : str or None
        The directory containing the `starfeatures-<objectid>.pkl` files for
        each object to use calculate neighbor proximity light curve features.

    fourierorder : int
        The Fourier order to use to generate sinusoidal function and fit that to
        the phased light curve.

    transitparams : list of floats
        The transit depth, duration, and ingress duration to use to generate a
        trapezoid planet transit model fit to the phased light curve. The period
        used is the one provided in `period`, while the epoch is automatically
        obtained from a spline fit to the phased light curve.

    ebparams : list of floats
        The primary eclipse depth, eclipse duration, the primary-secondary depth
        ratio, and the phase of the secondary eclipse to use to generate an
        eclipsing binary model fit to the phased light curve. The period used is
        the one provided in `period`, while the epoch is automatically obtained
        from a spline fit to the phased light curve.

    pdiff_threshold : float
        This is the max difference between periods to consider them the same.

    sidereal_threshold : float
        This is the max difference between any of the 'best' periods and the
        sidereal day periods to consider them the same.

    sampling_peak_multiplier : float
        This is the minimum multiplicative factor of a 'best' period's
        normalized periodogram peak over the sampling periodogram peak at the
        same period required to accept the 'best' period as possibly real.

    sampling_startp, sampling_endp : float
        If the `pgramlist` doesn't have a time-sampling Lomb-Scargle
        periodogram, it will be obtained automatically. Use these kwargs to
        control the minimum and maximum period interval to be searched when
        generating this periodogram.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate progress while working.

    maxobjects : int
        The total number of objects to process from `pfpkl_list`.

    nworkers : int
        The number of parallel workers to launch to process the input.

    Returns
    -------

    dict
        A dict containing key: val pairs of the input period-finder result and
        the output periodic feature result pickles for each input pickle is
        returned.

    '''
    # make sure to make the output directory if it doesn't exist
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    if maxobjects:
        pfpkl_list = pfpkl_list[:maxobjects]

    LOGINFO('%s periodfinding pickles to process' % len(pfpkl_list))

    # if the starfeaturedir is provided, try to find a starfeatures pickle for
    # each periodfinding pickle in pfpkl_list
    if starfeaturesdir and os.path.exists(starfeaturesdir):

        starfeatures_list = []

        LOGINFO('collecting starfeatures pickles...')

        for pfpkl in pfpkl_list:

            sfpkl1 = os.path.basename(pfpkl).replace('periodfinding',
                                                     'starfeatures')
            sfpkl2 = sfpkl1.replace('.gz','')

            sfpath1 = os.path.join(starfeaturesdir, sfpkl1)
            sfpath2 = os.path.join(starfeaturesdir, sfpkl2)

            if os.path.exists(sfpath1):
                starfeatures_list.append(sfpkl1)
            elif os.path.exists(sfpath2):
                starfeatures_list.append(sfpkl2)
            else:
                starfeatures_list.append(None)

    else:

        starfeatures_list = [None for x in pfpkl_list]

    # generate the task list
    kwargs = {'fourierorder':fourierorder,
              'transitparams':transitparams,
              'ebparams':ebparams,
              'pdiff_threshold':pdiff_threshold,
              'sidereal_threshold':sidereal_threshold,
              'sampling_peak_multiplier':sampling_peak_multiplier,
              'sampling_startp':sampling_startp,
              'sampling_endp':sampling_endp,
              'timecols':timecols,
              'magcols':magcols,
              'errcols':errcols,
              'lcformat':lcformat,
              'lcformatdir':lcformat,
              'sigclip':sigclip,
              'verbose':verbose}

    tasks = [(x, lcbasedir, outdir, y, kwargs) for (x,y) in
             zip(pfpkl_list, starfeatures_list)]

    LOGINFO('processing periodfinding pickles...')

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(_periodicfeatures_worker, tasks)

    results = [x for x in resultfutures]
    resdict = {os.path.basename(x):y for (x,y) in zip(pfpkl_list, results)}

    return resdict

This runs periodic feature generation in parallel for all periodfinding pickles in the input list .


def parallel_periodicfeatures_lcdir(
        pfpkl_dir,
        lcbasedir,
        outdir,
        pfpkl_glob='periodfinding-*.pkl*',
        starfeaturesdir=None,
        fourierorder=5,
        # these are depth, duration, ingress duration
        transitparams=(-0.01,0.1,0.1),
        # these are depth, duration, depth ratio, secphase
        ebparams=(-0.2,0.3,0.7,0.5),
        pdiff_threshold=1.0e-4,
        sidereal_threshold=1.0e-4,
        sampling_peak_multiplier=5.0,
        sampling_startp=None,
        sampling_endp=None,
        timecols=None,
        magcols=None,
        errcols=None,
        lcformat='hat-sql',
        lcformatdir=None,
        sigclip=10.0,
        verbose=False,
        maxobjects=None,
        nworkers=NCPUS,
        recursive=True,
):
    '''This runs parallel periodicfeature extraction for a directory of
    periodfinding result pickles.

    Parameters
    ----------

    pfpkl_dir : str
        The directory containing the pickles to process.

    lcbasedir : str
        The directory where all of the associated light curve files are located.

    outdir : str
        The directory where all the output will be written.

    pfpkl_glob : str
        The UNIX file glob to use to search for period-finder result pickles in
        `pfpkl_dir`.

    starfeaturesdir : str or None
        The directory containing the `starfeatures-<objectid>.pkl` files for
        each object to use calculate neighbor proximity light curve features.

    fourierorder : int
        The Fourier order to use to generate sinusoidal function and fit that to
        the phased light curve.

    transitparams : list of floats
        The transit depth, duration, and ingress duration to use to generate a
        trapezoid planet transit model fit to the phased light curve. The period
        used is the one provided in `period`, while the epoch is automatically
        obtained from a spline fit to the phased light curve.

    ebparams : list of floats
        The primary eclipse depth, eclipse duration, the primary-secondary depth
        ratio, and the phase of the secondary eclipse to use to generate an
        eclipsing binary model fit to the phased light curve. The period used is
        the one provided in `period`, while the epoch is automatically obtained
        from a spline fit to the phased light curve.

    pdiff_threshold : float
        This is the max difference between periods to consider them the same.

    sidereal_threshold : float
        This is the max difference between any of the 'best' periods and the
        sidereal day periods to consider them the same.

    sampling_peak_multiplier : float
        This is the minimum multiplicative factor of a 'best' period's
        normalized periodogram peak over the sampling periodogram peak at the
        same period required to accept the 'best' period as possibly real.

    sampling_startp, sampling_endp : float
        If the `pgramlist` doesn't have a time-sampling Lomb-Scargle
        periodogram, it will be obtained automatically. Use these kwargs to
        control the minimum and maximum period interval to be searched when
        generating this periodogram.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate progress while working.

    maxobjects : int
        The total number of objects to process from `pfpkl_list`.

    nworkers : int
        The number of parallel workers to launch to process the input.

    Returns
    -------

    dict
        A dict containing key: val pairs of the input period-finder result and
        the output periodic feature result pickles for each input pickle is
        returned.

    '''

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    fileglob = pfpkl_glob

    # now find the files
    LOGINFO('searching for periodfinding pickles in %s ...' % pfpkl_dir)

    if recursive is False:
        matching = glob.glob(os.path.join(pfpkl_dir, fileglob))

    else:
        # use recursive glob for Python 3.5+
        if sys.version_info[:2] > (3,4):

            matching = glob.glob(os.path.join(pfpkl_dir,
                                              '**',
                                              fileglob),recursive=True)

        # otherwise, use os.walk and glob
        else:

            # use os.walk to go through the directories
            walker = os.walk(pfpkl_dir)
            matching = []

            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              fileglob)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)

    # now that we have all the files, process them
    if matching and len(matching) > 0:

        LOGINFO('found %s periodfinding pickles, getting periodicfeatures...' %
                len(matching))

        return parallel_periodicfeatures(
            matching,
            lcbasedir,
            outdir,
            starfeaturesdir=starfeaturesdir,
            fourierorder=fourierorder,
            transitparams=transitparams,
            ebparams=ebparams,
            pdiff_threshold=pdiff_threshold,
            sidereal_threshold=sidereal_threshold,
            sampling_peak_multiplier=sampling_peak_multiplier,
            sampling_startp=sampling_startp,
            sampling_endp=sampling_endp,
            timecols=timecols,
            magcols=magcols,
            errcols=errcols,
            lcformat=lcformat,
            lcformatdir=lcformatdir,
            sigclip=sigclip,
            verbose=verbose,
            maxobjects=maxobjects,
            nworkers=nworkers,
        )

    else:

        LOGERROR('no periodfinding pickles found in %s' % (pfpkl_dir))
        return None

This runs parallel periodicfeature extraction for a directory of periodfinding result pickles .


def _parse_xmatch_catalog_header(xc, xk):
    '''
    This parses the header for a catalog file and returns it as a file object.

    Parameters
    ----------

    xc : str
        The file name of an xmatch catalog prepared previously.

    xk : list of str
        This is a list of column names to extract from the xmatch catalog.

    Returns
    -------

    tuple
        The tuple returned is of the form::

            (infd: the file object associated with the opened xmatch catalog,
             catdefdict: a dict describing the catalog column definitions,
             catcolinds: column number indices of the catalog,
             catcoldtypes: the numpy dtypes of the catalog columns,
             catcolnames: the names of each catalog column,
             catcolunits: the units associated with each catalog column)

    '''

    catdef = []

    # read in this catalog and transparently handle gzipped files
    if xc.endswith('.gz'):
        infd = gzip.open(xc,'rb')
    else:
        infd = open(xc,'rb')

    # read in the defs
    for line in infd:
        if line.decode().startswith('#'):
            catdef.append(
                line.decode().replace('#','').strip().rstrip('\n')
            )
        if not line.decode().startswith('#'):
            break

    if not len(catdef) > 0:
        LOGERROR("catalog definition not parseable "
                 "for catalog: %s, skipping..." % xc)
        return None

    catdef = ' '.join(catdef)
    catdefdict = json.loads(catdef)

    catdefkeys = [x['key'] for x in catdefdict['columns']]
    catdefdtypes = [x['dtype'] for x in catdefdict['columns']]
    catdefnames = [x['name'] for x in catdefdict['columns']]
    catdefunits = [x['unit'] for x in catdefdict['columns']]

    # get the correct column indices and dtypes for the requested columns
    # from the catdefdict

    catcolinds = []
    catcoldtypes = []
    catcolnames = []
    catcolunits = []

    for xkcol in xk:

        if xkcol in catdefkeys:

            xkcolind = catdefkeys.index(xkcol)

            catcolinds.append(xkcolind)
            catcoldtypes.append(catdefdtypes[xkcolind])
            catcolnames.append(catdefnames[xkcolind])
            catcolunits.append(catdefunits[xkcolind])

    return (infd, catdefdict,
            catcolinds, catcoldtypes, catcolnames, catcolunits)

This parses the header for a catalog file and returns it as a file object .


def load_xmatch_external_catalogs(xmatchto, xmatchkeys, outfile=None):
    '''This loads the external xmatch catalogs into a dict for use in an xmatch.

    Parameters
    ----------

    xmatchto : list of str
        This is a list of paths to all the catalog text files that will be
        loaded.

        The text files must be 'CSVs' that use the '|' character as the
        separator betwen columns. These files should all begin with a header in
        JSON format on lines starting with the '#' character. this header will
        define the catalog and contains the name of the catalog and the column
        definitions. Column definitions must have the column name and the numpy
        dtype of the columns (in the same format as that expected for the
        numpy.genfromtxt function). Any line that does not begin with '#' is
        assumed to be part of the columns in the catalog. An example is shown
        below::

            # {"name":"NSVS catalog of variable stars",
            #  "columns":[
            #   {"key":"objectid", "dtype":"U20", "name":"Object ID", "unit": null},
            #   {"key":"ra", "dtype":"f8", "name":"RA", "unit":"deg"},
            #   {"key":"decl","dtype":"f8", "name": "Declination", "unit":"deg"},
            #   {"key":"sdssr","dtype":"f8","name":"SDSS r", "unit":"mag"},
            #   {"key":"vartype","dtype":"U20","name":"Variable type", "unit":null}
            #  ],
            #  "colra":"ra",
            #  "coldec":"decl",
            #  "description":"Contains variable stars from the NSVS catalog"}
            objectid1 | 45.0  | -20.0 | 12.0 | detached EB
            objectid2 | 145.0 | 23.0  | 10.0 | RRab
            objectid3 | 12.0  | 11.0  | 14.0 | Cepheid
            .
            .
            .

    xmatchkeys : list of lists
        This is the list of lists of column names (as str) to get out of each
        `xmatchto` catalog. This should be the same length as `xmatchto` and
        each element here will apply to the respective file in `xmatchto`.

    outfile : str or None
        If this is not None, set this to the name of the pickle to write the
        collected xmatch catalogs to. this pickle can then be loaded
        transparently by the :py:func:`astrobase.checkplot.pkl.checkplot_dict`,
        :py:func:`astrobase.checkplot.pkl.checkplot_pickle` functions to provide
        xmatch info to the
        :py:func:`astrobase.checkplot.pkl_xmatch.xmatch_external_catalogs`
        function below.

        If this is None, will return the loaded xmatch catalogs directly. This
        will be a huge dict, so make sure you have enough RAM.

    Returns
    -------

    str or dict
        Based on the `outfile` kwarg, will either return the path to a collected
        xmatch pickle file or the collected xmatch dict.

    '''

    outdict = {}

    for xc, xk in zip(xmatchto, xmatchkeys):

        parsed_catdef = _parse_xmatch_catalog_header(xc, xk)

        if not parsed_catdef:
            continue

        (infd, catdefdict,
         catcolinds, catcoldtypes,
         catcolnames, catcolunits) = parsed_catdef

        # get the specified columns out of the catalog
        catarr = np.genfromtxt(infd,
                               usecols=catcolinds,
                               names=xk,
                               dtype=','.join(catcoldtypes),
                               comments='#',
                               delimiter='|',
                               autostrip=True)
        infd.close()

        catshortname = os.path.splitext(os.path.basename(xc))[0]
        catshortname = catshortname.replace('.csv','')

        #
        # make a kdtree for this catalog
        #

        # get the ra and decl columns
        objra, objdecl = (catarr[catdefdict['colra']],
                          catarr[catdefdict['coldec']])

        # get the xyz unit vectors from ra,decl
        cosdecl = np.cos(np.radians(objdecl))
        sindecl = np.sin(np.radians(objdecl))
        cosra = np.cos(np.radians(objra))
        sinra = np.sin(np.radians(objra))
        xyz = np.column_stack((cosra*cosdecl,sinra*cosdecl, sindecl))

        # generate the kdtree
        kdt = cKDTree(xyz,copy_data=True)

        # generate the outdict element for this catalog
        catoutdict = {'kdtree':kdt,
                      'data':catarr,
                      'columns':xk,
                      'colnames':catcolnames,
                      'colunits':catcolunits,
                      'name':catdefdict['name'],
                      'desc':catdefdict['description']}

        outdict[catshortname] = catoutdict

    if outfile is not None:

        # if we're on OSX, we apparently need to save the file in chunks smaller
        # than 2 GB to make it work right. can't load pickles larger than 4 GB
        # either, but 3 GB < total size < 4 GB appears to be OK when loading.
        # also see: https://bugs.python.org/issue24658.
        # fix adopted from: https://stackoverflow.com/a/38003910
        if sys.platform == 'darwin':

            dumpbytes = pickle.dumps(outdict, protocol=pickle.HIGHEST_PROTOCOL)
            max_bytes = 2**31 - 1

            with open(outfile, 'wb') as outfd:
                for idx in range(0, len(dumpbytes), max_bytes):
                    outfd.write(dumpbytes[idx:idx+max_bytes])

        else:
            with open(outfile, 'wb') as outfd:
                pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

        return outfile

    else:

        return outdict

This loads the external xmatch catalogs into a dict for use in an xmatch .


def angle_wrap(angle, radians=False):
    '''Wraps the input angle to 360.0 degrees.

    Parameters
    ----------

    angle : float
        The angle to wrap around 360.0 deg.

    radians : bool
        If True, will assume that the input is in radians. The output will then
        also be in radians.

    Returns
    -------

    float
        Wrapped angle. If radians is True: input is assumed to be in radians,
        output is also in radians.

    '''

    if radians:
        wrapped = angle % (2.0*pi_value)
        if wrapped < 0.0:
            wrapped = 2.0*pi_value + wrapped

    else:

        wrapped = angle % 360.0
        if wrapped < 0.0:
            wrapped = 360.0 + wrapped

    return wrapped

def angle_wrap ( angle , radians = False ) : if radians : wrapped = angle % ( 2.0 * pi_value ) if wrapped < 0.0 : wrapped = 2.0 * pi_value + wrapped else : wrapped = angle % 360.0 if wrapped < 0.0 : wrapped = 360.0 + wrapped return wrapped

Wraps the input angle to 360 . 0 degrees .


def hms_to_decimal(hours, minutes, seconds, returndeg=True):
    '''Converts from HH, MM, SS to a decimal value.

    Parameters
    ----------

    hours : int
        The HH part of a RA coordinate.

    minutes : int
        The MM part of a RA coordinate.

    seconds : float
        The SS.sss part of a RA coordinate.

    returndeg : bool
        If this is True, then will return decimal degrees as the output.
        If this is False, then will return decimal HOURS as the output.
        Decimal hours are sometimes used in FITS headers.

    Returns
    -------

    float
        The right ascension value in either decimal degrees or decimal hours
        depending on `returndeg`.

    '''

    if hours > 24:

        return None

    else:

        dec_hours = fabs(hours) + fabs(minutes)/60.0 + fabs(seconds)/3600.0

        if returndeg:

            dec_deg = dec_hours*15.0

            if dec_deg < 0:
                dec_deg = dec_deg + 360.0
            dec_deg = dec_deg % 360.0
            return dec_deg
        else:
            return dec_hours

def hms_to_decimal ( hours , minutes , seconds , returndeg = True ) : if hours > 24 : return None else : dec_hours = fabs ( hours ) + fabs ( minutes ) / 60.0 + fabs ( seconds ) / 3600.0 if returndeg : dec_deg = dec_hours * 15.0 if dec_deg < 0 : dec_deg = dec_deg + 360.0 dec_deg = dec_deg % 360.0 return dec_deg else : return dec_hours

Converts from HH MM SS to a decimal value .


def great_circle_dist(ra1, dec1, ra2, dec2):
    '''Calculates the great circle angular distance between two coords.

    This calculates the great circle angular distance in arcseconds between two
    coordinates (ra1,dec1) and (ra2,dec2). This is basically a clone of GCIRC
    from the IDL Astrolib.

    Parameters
    ----------

    ra1,dec1 : float or array-like
        The first coordinate's right ascension and declination value(s) in
        decimal degrees.

    ra2,dec2 : float or array-like
        The second coordinate's right ascension and declination value(s) in
        decimal degrees.

    Returns
    -------

    float or array-like
        Great circle distance between the two coordinates in arseconds.

    Notes
    -----

    If (`ra1`, `dec1`) is scalar and (`ra2`, `dec2`) is scalar: the result is a
    float distance in arcseconds.

    If (`ra1`, `dec1`) is scalar and (`ra2`, `dec2`) is array-like: the result
    is an np.array with distance in arcseconds between (`ra1`, `dec1`) and each
    element of (`ra2`, `dec2`).

    If (`ra1`, `dec1`) is array-like and (`ra2`, `dec2`) is scalar: the result
    is an np.array with distance in arcseconds between (`ra2`, `dec2`) and each
    element of (`ra1`, `dec1`).

    If (`ra1`, `dec1`) and (`ra2`, `dec2`) are both array-like: the result is an
    np.array with the pair-wise distance in arcseconds between each element of
    the two coordinate lists. In this case, if the input array-likes are not the
    same length, then excess elements of the longer one will be ignored.

    '''

    # wrap RA if negative or larger than 360.0 deg
    in_ra1 = ra1 % 360.0
    in_ra1 = in_ra1 + 360.0*(in_ra1 < 0.0)
    in_ra2 = ra2 % 360.0
    in_ra2 = in_ra2 + 360.0*(in_ra1 < 0.0)

    # convert to radians
    ra1_rad, dec1_rad = np.deg2rad(in_ra1), np.deg2rad(dec1)
    ra2_rad, dec2_rad = np.deg2rad(in_ra2), np.deg2rad(dec2)

    del_dec2 = (dec2_rad - dec1_rad)/2.0
    del_ra2 = (ra2_rad - ra1_rad)/2.0
    sin_dist = np.sqrt(np.sin(del_dec2) * np.sin(del_dec2) +
                       np.cos(dec1_rad) * np.cos(dec2_rad) *
                       np.sin(del_ra2) * np.sin(del_ra2))

    dist_rad = 2.0 * np.arcsin(sin_dist)

    # return the distance in arcseconds
    return np.rad2deg(dist_rad)*3600.0

Calculates the great circle angular distance between two coords .


def total_proper_motion(pmra, pmdecl, decl):

    '''This calculates the total proper motion of an object.

    Parameters
    ----------

    pmra : float or array-like
        The proper motion(s) in right ascension, measured in mas/yr.

    pmdecl : float or array-like
        The proper motion(s) in declination, measured in mas/yr.

    decl : float or array-like
        The declination of the object(s) in decimal degrees.

    Returns
    -------

    float or array-like
        The total proper motion(s) of the object(s) in mas/yr.

    '''

    pm = np.sqrt( pmdecl*pmdecl + pmra*pmra*np.cos(np.radians(decl)) *
                  np.cos(np.radians(decl)) )
    return pm

def total_proper_motion ( pmra , pmdecl , decl ) : pm = np . sqrt ( pmdecl * pmdecl + pmra * pmra * np . cos ( np . radians ( decl ) ) * np . cos ( np . radians ( decl ) ) ) return pm

This calculates the total proper motion of an object .


def equatorial_to_galactic(ra, decl, equinox='J2000'):
    '''This converts from equatorial coords to galactic coords.

    Parameters
    ----------

    ra : float or array-like
        Right ascension values(s) in decimal degrees.

    decl : float or array-like
        Declination value(s) in decimal degrees.

    equinox : str
        The equinox that the coordinates are measured at. This must be
        recognizable by Astropy's `SkyCoord` class.

    Returns
    -------

    tuple of (float, float) or tuple of (np.array, np.array)
        The galactic coordinates (l, b) for each element of the input
        (`ra`, `decl`).

    '''

    # convert the ra/decl to gl, gb
    radecl = SkyCoord(ra=ra*u.degree, dec=decl*u.degree, equinox=equinox)

    gl = radecl.galactic.l.degree
    gb = radecl.galactic.b.degree

    return gl, gb

def
equatorial_to_galactic
(
ra
,
decl
,
equinox
=
'J2000'
)
:
# convert the ra/decl to gl, gb
radecl
=
SkyCoord
(
ra
=
ra
*
u
.
degree
,
dec
=
decl
*
u
.
degree
,
equinox
=
equinox
)
gl
=
radecl
.
galactic
.
l
.
degree
gb
=
radecl
.
galactic
.
b
.
degree
return
gl
,
gb

This converts from equatorial coords to galactic coords .


def galactic_to_equatorial(gl, gb):
    '''This converts from galactic coords to equatorial coordinates.

    Parameters
    ----------

    gl : float or array-like
        Galactic longitude values(s) in decimal degrees.

    gb : float or array-like
        Galactic latitude value(s) in decimal degrees.

    Returns
    -------

    tuple of (float, float) or tuple of (np.array, np.array)
        The equatorial coordinates (RA, DEC) for each element of the input
        (`gl`, `gb`) in decimal degrees. These are reported in the ICRS frame.

    '''

    gal = SkyCoord(gl*u.degree, gl*u.degree, frame='galactic')

    transformed = gal.transform_to('icrs')

    return transformed.ra.degree, transformed.dec.degree

def galactic_to_equatorial ( gl , gb ) : gal = SkyCoord ( gl * u . degree , gl * u . degree , frame = 'galactic' ) transformed = gal . transform_to ( 'icrs' ) return transformed . ra . degree , transformed . dec . degree

This converts from galactic coords to equatorial coordinates .


def xieta_from_radecl(inra, indecl,
                      incenterra, incenterdecl,
                      deg=True):
    '''This returns the image-plane projected xi-eta coords for inra, indecl.

    Parameters
    ----------

    inra,indecl : array-like
        The equatorial coordinates to get the xi, eta coordinates for in decimal
        degrees or radians.

    incenterra,incenterdecl : float
        The center coordinate values to use to calculate the plane-projected
        coordinates around.

    deg : bool
        If this is True, the input angles are assumed to be in degrees and the
        output is in degrees as well.

    Returns
    -------

    tuple of np.arrays
        This is the (`xi`, `eta`) coordinate pairs corresponding to the
        image-plane projected coordinates for each pair of input equatorial
        coordinates in (`inra`, `indecl`).

    '''

    if deg:

        ra = np.radians(inra)
        decl = np.radians(indecl)
        centerra = np.radians(incenterra)
        centerdecl = np.radians(incenterdecl)

    else:

        ra = inra
        decl = indecl
        centerra = incenterra
        centerdecl = incenterdecl

    cdecc = np.cos(centerdecl)
    sdecc = np.sin(centerdecl)
    crac = np.cos(centerra)
    srac = np.sin(centerra)

    uu = np.cos(decl)*np.cos(ra)
    vv = np.cos(decl)*np.sin(ra)
    ww = np.sin(decl)

    uun = uu*cdecc*crac + vv*cdecc*srac + ww*sdecc
    vvn = -uu*srac + vv*crac
    wwn = -uu*sdecc*crac - vv*sdecc*srac + ww*cdecc
    denom = vvn*vvn + wwn*wwn

    aunn = np.zeros_like(uun)
    aunn[uun >= 1.0] = 0.0
    aunn[uun < 1.0] = np.arccos(uun)

    xi, eta = np.zeros_like(aunn), np.zeros_like(aunn)

    xi[(aunn <= 0.0) | (denom <= 0.0)] = 0.0
    eta[(aunn <= 0.0) | (denom <= 0.0)] = 0.0

    sdenom = np.sqrt(denom)

    xi[(aunn > 0.0) | (denom > 0.0)] = aunn*vvn/sdenom
    eta[(aunn > 0.0) | (denom > 0.0)] = aunn*wwn/sdenom

    if deg:
        return np.degrees(xi), np.degrees(eta)
    else:
        return xi, eta

def xieta_from_radecl ( inra , indecl , incenterra , incenterdecl , deg = True ) : if deg : ra = np . radians ( inra ) decl = np . radians ( indecl ) centerra = np . radians ( incenterra ) centerdecl = np . radians ( incenterdecl ) else : ra = inra decl = indecl centerra = incenterra centerdecl = incenterdecl cdecc = np . cos ( centerdecl ) sdecc = np . sin ( centerdecl ) crac = np . cos ( centerra ) srac = np . sin ( centerra ) uu = np . cos ( decl ) * np . cos ( ra ) vv = np . cos ( decl ) * np . sin ( ra ) ww = np . sin ( decl ) uun = uu * cdecc * crac + vv * cdecc * srac + ww * sdecc vvn = - uu * srac + vv * crac wwn = - uu * sdecc * crac - vv * sdecc * srac + ww * cdecc denom = vvn * vvn + wwn * wwn aunn = np . zeros_like ( uun ) aunn [ uun >= 1.0 ] = 0.0 aunn [ uun < 1.0 ] = np . arccos ( uun ) xi , eta = np . zeros_like ( aunn ) , np . zeros_like ( aunn ) xi [ ( aunn <= 0.0 ) | ( denom <= 0.0 ) ] = 0.0 eta [ ( aunn <= 0.0 ) | ( denom <= 0.0 ) ] = 0.0 sdenom = np . sqrt ( denom ) xi [ ( aunn > 0.0 ) | ( denom > 0.0 ) ] = aunn * vvn / sdenom eta [ ( aunn > 0.0 ) | ( denom > 0.0 ) ] = aunn * wwn / sdenom if deg : return np . degrees ( xi ) , np . degrees ( eta ) else : return xi , eta

This returns the image - plane projected xi - eta coords for inra indecl .


def generate_transit_lightcurve(
        times,
        mags=None,
        errs=None,
        paramdists={'transitperiod':sps.uniform(loc=0.1,scale=49.9),
                    'transitdepth':sps.uniform(loc=1.0e-4,scale=2.0e-2),
                    'transitduration':sps.uniform(loc=0.01,scale=0.29)},
        magsarefluxes=False,
):
    '''This generates fake planet transit light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'transitperiod', 'transitdepth', 'transitduration'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The ingress duration will be automatically chosen from a uniform
        distribution ranging from 0.05 to 0.5 of the transitduration.

        The transitdepth will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'planet',
             'params': {'transitperiod': generated value of period,
                        'transitepoch': generated value of epoch,
                        'transitdepth': generated value of transit depth,
                        'transitduration': generated value of transit duration,
                        'ingressduration': generated value of transit ingress
                                           duration},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'transitperiod'
             'varamplitude': the generated amplitude of
                             variability == 'transitdepth'}

    '''

    if mags is None:
        mags = np.full_like(times, 0.0)

    if errs is None:
        errs = np.full_like(times, 0.0)

    # choose the epoch
    epoch = npr.random()*(times.max() - times.min()) + times.min()

    # choose the period, depth, duration
    period = paramdists['transitperiod'].rvs(size=1)
    depth = paramdists['transitdepth'].rvs(size=1)
    duration = paramdists['transitduration'].rvs(size=1)

    # figure out the ingress duration
    ingduration = npr.random()*(0.5*duration - 0.05*duration) + 0.05*duration

    # fix the transit depth if it needs to be flipped
    if magsarefluxes and depth < 0.0:
        depth = -depth
    elif not magsarefluxes and depth > 0.0:
        depth = -depth

    # generate the model
    modelmags, phase, ptimes, pmags, perrs = (
        transits.trapezoid_transit_func([period, epoch, depth,
                                         duration, ingduration],
                                        times,
                                        mags,
                                        errs)
    )

    # resort in original time order
    timeind = np.argsort(ptimes)
    mtimes = ptimes[timeind]
    mmags = modelmags[timeind]
    merrs = perrs[timeind]

    # return a dict with everything
    modeldict = {
        'vartype':'planet',
        'params':{x:np.asscalar(y) for x,y in zip(['transitperiod',
                                                   'transitepoch',
                                                   'transitdepth',
                                                   'transitduration',
                                                   'ingressduration'],
                                                  [period,
                                                   epoch,
                                                   depth,
                                                   duration,
                                                   ingduration])},
        'times':mtimes,
        'mags':mmags,
        'errs':merrs,
        # these are standard keys that help with later characterization of
        # variability as a function period, variability amplitude, object mag,
        # ndet, etc.
        'varperiod':period,
        'varamplitude':depth
    }

    return modeldict

This generates fake planet transit light curves .


def generate_eb_lightcurve(
        times,
        mags=None,
        errs=None,
        paramdists={'period':sps.uniform(loc=0.2,scale=99.8),
                    'pdepth':sps.uniform(loc=1.0e-4,scale=0.7),
                    'pduration':sps.uniform(loc=0.01,scale=0.44),
                    'depthratio':sps.uniform(loc=0.01,scale=0.99),
                    'secphase':sps.norm(loc=0.5,scale=0.1)},
        magsarefluxes=False,
):
    '''This generates fake EB light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'pdepth', 'pduration', 'depthratio', 'secphase'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `pdepth` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'EB',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'pdepth': generated value of priary eclipse depth,
                        'pduration': generated value of prim eclipse duration,
                        'depthratio': generated value of prim/sec eclipse
                                      depth ratio},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'pdepth'}

    '''

    if mags is None:
        mags = np.full_like(times, 0.0)

    if errs is None:
        errs = np.full_like(times, 0.0)

    # choose the epoch
    epoch = npr.random()*(times.max() - times.min()) + times.min()

    # choose the period, pdepth, duration, depthratio
    period = paramdists['period'].rvs(size=1)
    pdepth = paramdists['pdepth'].rvs(size=1)
    pduration = paramdists['pduration'].rvs(size=1)
    depthratio = paramdists['depthratio'].rvs(size=1)
    secphase = paramdists['secphase'].rvs(size=1)

    # fix the transit depth if it needs to be flipped
    if magsarefluxes and pdepth < 0.0:
        pdepth = -pdepth
    elif not magsarefluxes and pdepth > 0.0:
        pdepth = -pdepth

    # generate the model
    modelmags, phase, ptimes, pmags, perrs = (
        eclipses.invgauss_eclipses_func([period, epoch, pdepth,
                                         pduration, depthratio, secphase],
                                        times,
                                        mags,
                                        errs)
    )

    # resort in original time order
    timeind = np.argsort(ptimes)
    mtimes = ptimes[timeind]
    mmags = modelmags[timeind]
    merrs = perrs[timeind]

    # return a dict with everything
    modeldict = {
        'vartype':'EB',
        'params':{x:np.asscalar(y) for x,y in zip(['period',
                                                   'epoch',
                                                   'pdepth',
                                                   'pduration',
                                                   'depthratio'],
                                                  [period,
                                                   epoch,
                                                   pdepth,
                                                   pduration,
                                                   depthratio])},
        'times':mtimes,
        'mags':mmags,
        'errs':merrs,
        'varperiod':period,
        'varamplitude':pdepth,
    }

    return modeldict


def generate_flare_lightcurve(
        times,
        mags=None,
        errs=None,
        paramdists={
            # flare peak amplitude from 0.01 mag to 1.0 mag above median.  this
            # is tuned for redder bands, flares are much stronger in bluer
            # bands, so tune appropriately for your situation.
            'amplitude':sps.uniform(loc=0.01,scale=0.99),
            # up to 5 flares per LC and at least 1
            'nflares':[1,5],
            # 10 minutes to 1 hour for rise stdev
            'risestdev':sps.uniform(loc=0.007, scale=0.04),
            # 1 hour to 4 hours for decay time constant
            'decayconst':sps.uniform(loc=0.04, scale=0.163)
        },
        magsarefluxes=False,
):
    '''This generates fake flare light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'amplitude', 'nflares', 'risestdev', 'decayconst'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The `flare_peak_time` for each flare will be generated automatically
        between `times.min()` and `times.max()` using a uniform distribution.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'flare',
             'params': {'amplitude': generated value of flare amplitudes,
                        'nflares': generated value of number of flares,
                        'risestdev': generated value of stdev of rise time,
                        'decayconst': generated value of decay constant,
                        'peaktime': generated value of flare peak time},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}

    '''

    if mags is None:
        mags = np.full_like(times, 0.0)

    if errs is None:
        errs = np.full_like(times, 0.0)

    nflares = npr.randint(paramdists['nflares'][0],
                          high=paramdists['nflares'][1])

    # generate random flare peak times based on the number of flares
    flarepeaktimes = (
        npr.random(
            size=nflares
        )*(times.max() - times.min()) + times.min()
    )

    # now add the flares to the time-series
    params = {'nflares':nflares}

    for flareind, peaktime in zip(range(nflares), flarepeaktimes):

        # choose the amplitude, rise stdev and decay time constant
        amp = paramdists['amplitude'].rvs(size=1)
        risestdev = paramdists['risestdev'].rvs(size=1)
        decayconst = paramdists['decayconst'].rvs(size=1)

        # fix the transit depth if it needs to be flipped
        if magsarefluxes and amp < 0.0:
            amp = -amp
        elif not magsarefluxes and amp > 0.0:
            amp = -amp

        # add this flare to the light curve
        modelmags, ptimes, pmags, perrs = (
            flares.flare_model(
                [amp, peaktime, risestdev, decayconst],
                times,
                mags,
                errs
            )
        )

        # update the mags
        mags = modelmags

        # add the flare params to the modeldict
        params[flareind] = {'peaktime':peaktime,
                            'amplitude':amp,
                            'risestdev':risestdev,
                            'decayconst':decayconst}

    #
    # done with all flares
    #

    # return a dict with everything
    modeldict = {
        'vartype':'flare',
        'params':params,
        'times':times,
        'mags':mags,
        'errs':errs,
        'varperiod':None,
        # FIXME: this is complicated because we can have multiple flares
        # figure out a good way to handle this upstream
        'varamplitude':[params[x]['amplitude']
                        for x in range(params['nflares'])],
    }

    return modeldict

This generates fake flare light curves .


def generate_sinusoidal_lightcurve(
        times,
        mags=None,
        errs=None,
        paramdists={
            'period':sps.uniform(loc=0.04,scale=500.0),
            'fourierorder':[2,10],
            'amplitude':sps.uniform(loc=0.1,scale=0.9),
            'phioffset':0.0,
        },
        magsarefluxes=False
):
    '''This generates fake sinusoidal light curves.

    This can be used for a variety of sinusoidal variables, e.g. RRab, RRc,
    Cepheids, Miras, etc. The functions that generate these model LCs below
    implement the following table::

        ## FOURIER PARAMS FOR SINUSOIDAL VARIABLES
        #
        # type        fourier           period [days]
        #             order    dist     limits         dist

        # RRab        8 to 10  uniform  0.45--0.80     uniform
        # RRc         3 to 6   uniform  0.10--0.40     uniform
        # HADS        7 to 9   uniform  0.04--0.10     uniform
        # rotator     2 to 5   uniform  0.80--120.0    uniform
        # LPV         2 to 5   uniform  250--500.0     uniform

    FIXME: for better model LCs, figure out how scipy.signal.butter works and
    low-pass filter using scipy.signal.filtfilt.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude', 'phioffset'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'sinusoidal',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}

    '''

    if mags is None:
        mags = np.full_like(times, 0.0)

    if errs is None:
        errs = np.full_like(times, 0.0)

    # choose the epoch
    epoch = npr.random()*(times.max() - times.min()) + times.min()

    # choose the period, fourierorder, and amplitude
    period = paramdists['period'].rvs(size=1)
    fourierorder = npr.randint(paramdists['fourierorder'][0],
                               high=paramdists['fourierorder'][1])
    amplitude = paramdists['amplitude'].rvs(size=1)

    # fix the amplitude if it needs to be flipped
    if magsarefluxes and amplitude < 0.0:
        amplitude = -amplitude
    elif not magsarefluxes and amplitude > 0.0:
        amplitude = -amplitude

    # generate the amplitudes and phases of the Fourier components
    ampcomps = [abs(amplitude/2.0)/float(x)
                for x in range(1,fourierorder+1)]
    phacomps = [paramdists['phioffset']*float(x)
                for x in range(1,fourierorder+1)]

    # now that we have our amp and pha components, generate the light curve
    modelmags, phase, ptimes, pmags, perrs = sinusoidal.sine_series_sum(
        [period, epoch, ampcomps, phacomps],
        times,
        mags,
        errs
    )

    # resort in original time order
    timeind = np.argsort(ptimes)
    mtimes = ptimes[timeind]
    mmags = modelmags[timeind]
    merrs = perrs[timeind]
    mphase = phase[timeind]

    # return a dict with everything
    modeldict = {
        'vartype':'sinusoidal',
        'params':{x:y for x,y in zip(['period',
                                      'epoch',
                                      'amplitude',
                                      'fourierorder',
                                      'fourieramps',
                                      'fourierphases'],
                                     [period,
                                      epoch,
                                      amplitude,
                                      fourierorder,
                                      ampcomps,
                                      phacomps])},
        'times':mtimes,
        'mags':mmags,
        'errs':merrs,
        'phase':mphase,
        # these are standard keys that help with later characterization of
        # variability as a function period, variability amplitude, object mag,
        # ndet, etc.
        'varperiod':period,
        'varamplitude':amplitude
    }

    return modeldict

This generates fake sinusoidal light curves .


def generate_rrab_lightcurve(
        times,
        mags=None,
        errs=None,
        paramdists={
            'period':sps.uniform(loc=0.45,scale=0.35),
            'fourierorder':[8,11],
            'amplitude':sps.uniform(loc=0.4,scale=0.5),
            'phioffset':np.pi,
        },
        magsarefluxes=False
):
    '''This generates fake RRab light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'RRab',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}

    '''

    modeldict = generate_sinusoidal_lightcurve(times,
                                               mags=mags,
                                               errs=errs,
                                               paramdists=paramdists,
                                               magsarefluxes=magsarefluxes)
    modeldict['vartype'] = 'RRab'
    return modeldict

def generate_rrab_lightcurve ( times , mags = None , errs = None , paramdists = { 'period' : sps . uniform ( loc = 0.45 , scale = 0.35 ) , 'fourierorder' : [ 8 , 11 ] , 'amplitude' : sps . uniform ( loc = 0.4 , scale = 0.5 ) , 'phioffset' : np . pi , } , magsarefluxes = False ) : modeldict = generate_sinusoidal_lightcurve ( times , mags = mags , errs = errs , paramdists = paramdists , magsarefluxes = magsarefluxes ) modeldict [ 'vartype' ] = 'RRab' return modeldict


def collection_worker(task):
    '''
    This wraps `process_fakelc` for `make_fakelc_collection` below.

    Parameters
    ----------

    task : tuple
        This is of the form::

            task[0] = lcfile
            task[1] = outdir
            task[2] = magrms
            task[3] = dict with keys: {'lcformat', 'timecols', 'magcols',
                                       'errcols', 'randomizeinfo'}

    Returns
    -------

    tuple
        This returns a tuple of the form::

            (fakelc_fpath,
             fakelc_lcdict['columns'],
             fakelc_lcdict['objectinfo'],
             fakelc_lcdict['moments'])
    '''

    lcfile, outdir, kwargs = task

    try:

        fakelcresults = make_fakelc(
            lcfile,
            outdir,
            **kwargs
        )

        return fakelcresults

    except Exception as e:

        LOGEXCEPTION('could not process %s into a fakelc' % lcfile)
        return None

def
collection_worker
(
task
)
:
lcfile
,
outdir
,
kwargs
=
task
try
:
fakelcresults
=
make_fakelc
(
lcfile
,
outdir
,
*
*
kwargs
)
return
fakelcresults
except
Exception
as
e
:
LOGEXCEPTION
(
'could not process %s into a fakelc'
%
lcfile
)
return

This wraps process_fakelc for make_fakelc_collection below .