def bls_stats_singleperiod(times, mags, errs, period,
                           magsarefluxes=False,
                           sigclip=10.0,
                           perioddeltapercent=10,
                           nphasebins=200,
                           mintransitduration=0.01,
                           maxtransitduration=0.4,
                           ingressdurationfraction=0.1,
                           verbose=True):
    '''This calculates the SNR, depth, duration, a refit period, and time of
    center-transit for a single period.

    The equation used for SNR is::

        SNR = (transit model depth / RMS of LC with transit model subtracted)
              * sqrt(number of points in transit)

    NOTE: you should set the kwargs `sigclip`, `nphasebins`,
    `mintransitduration`, `maxtransitduration` to what you used for an initial
    BLS run to detect transits in the input light curve to match those input
    conditions.

    Parameters
    ----------

    times,mags,errs : np.array
        These contain the magnitude/flux time-series and any associated errors.

    period : float
        The period to search around and refit the transits. This will be used to
        calculate the start and end periods of a rerun of BLS to calculate the
        stats.

    magsarefluxes : bool
        Set to True if the input measurements in `mags` are actually fluxes and
        not magnitudes.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    perioddeltapercent : float
        The fraction of the period provided to use to search around this
        value. This is a percentage. The period range searched will then be::

            [period - (perioddeltapercent/100.0)*period,
             period + (perioddeltapercent/100.0)*period]

    nphasebins : int
        The number of phase bins to use in the BLS run.

    mintransitduration : float
        The minimum transit duration in phase to consider.

    maxtransitduration : float
        The maximum transit duration to consider.

    ingressdurationfraction : float
        The fraction of the transit duration to use to generate an initial value
        of the transit ingress duration for the BLS model refit. This will be
        fit by this function.

    verbose : bool
        If True, will indicate progress and any problems encountered.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'period': the refit best period,
             'epoch': the refit epoch (i.e. mid-transit time),
             'snr':the SNR of the transit,
             'transitdepth':the depth of the transit,
             'transitduration':the duration of the transit,
             'nphasebins':the input value of nphasebins,
             'transingressbin':the phase bin containing transit ingress,
             'transegressbin':the phase bin containing transit egress,
             'blsmodel':the full BLS model used along with its parameters,
             'subtractedmags':BLS model - phased light curve,
             'phasedmags':the phase light curve,
             'phases': the phase values}

    '''

    # get rid of nans first and sigclip
    stimes, smags, serrs = sigclip_magseries(times,
                                             mags,
                                             errs,
                                             magsarefluxes=magsarefluxes,
                                             sigclip=sigclip)

    # make sure there are enough points to calculate a spectrum
    if len(stimes) > 9 and len(smags) > 9 and len(serrs) > 9:

        # get the period interval
        startp = period - perioddeltapercent*period/100.0

        if startp < 0:
            startp = period

        endp = period + perioddeltapercent*period/100.0

        # rerun BLS in serial mode around the specified period to get the
        # transit depth, duration, ingress and egress bins
        blsres = bls_serial_pfind(stimes, smags, serrs,
                                  verbose=verbose,
                                  startp=startp,
                                  endp=endp,
                                  nphasebins=nphasebins,
                                  mintransitduration=mintransitduration,
                                  maxtransitduration=maxtransitduration,
                                  magsarefluxes=magsarefluxes,
                                  get_stats=False,
                                  sigclip=None)

        if (not blsres or
            'blsresult' not in blsres or
            blsres['blsresult'] is None):
            LOGERROR("BLS failed during a period-search "
                     "performed around the input best period: %.6f. "
                     "Can't continue. " % period)
            return None

        thistransdepth = blsres['blsresult']['transdepth']
        thistransduration = blsres['blsresult']['transduration']
        thisbestperiod = blsres['bestperiod']
        thistransingressbin = blsres['blsresult']['transingressbin']
        thistransegressbin = blsres['blsresult']['transegressbin']
        thisnphasebins = nphasebins

        stats = _get_bls_stats(stimes,
                               smags,
                               serrs,
                               thistransdepth,
                               thistransduration,
                               ingressdurationfraction,
                               nphasebins,
                               thistransingressbin,
                               thistransegressbin,
                               thisbestperiod,
                               thisnphasebins,
                               magsarefluxes=magsarefluxes,
                               verbose=verbose)

        return stats

    # if there aren't enough points in the mag series, bail out
    else:

        LOGERROR('no good detections for these times and mags, skipping...')
        return None

This calculates the SNR depth duration a refit period and time of center - transit for a single period .


def massradius(age, planetdist, coremass,
               mass='massjupiter',
               radius='radiusjupiter'):
    '''This function gets the Fortney mass-radius relation for planets.

    Parameters
    ----------

    age : float
        This should be one of: 0.3, 1.0, 4.5 [in Gyr].

    planetdist : float
        This should be one of: 0.02, 0.045, 0.1, 1.0, 9.5 [in AU]

    coremass : int
        This should be one of: 0, 10, 25, 50, 100 [in Mearth]

    mass : {'massjupiter','massearth'}
        Sets the mass units.

    radius : str
        Sets the radius units. Only 'radiusjupiter' is used for now.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'mass': an array containing the masses to plot),
             'radius': an array containing the radii to plot}

        These can be passed to a plotting routine to make mass-radius plot for
        the specified age, planet-star distance, and core-mass.

    '''

    MR = {0.3:MASSESRADII_0_3GYR,
          1.0:MASSESRADII_1_0GYR,
          4.5:MASSESRADII_4_5GYR}

    if age not in MR:
        print('given age not in Fortney 2007, returning...')
        return

    massradius = MR[age]

    if (planetdist in massradius) and (coremass in massradius[planetdist]):

        print('getting % Gyr M-R for planet dist %s AU, '
              'core mass %s Mearth...' % (age, planetdist, coremass))

        massradrelation = massradius[planetdist][coremass]

        outdict = {'mass':array(massradrelation[mass]),
                   'radius':array(massradrelation[radius])}

        return outdict

This function gets the Fortney mass - radius relation for planets .


def _reform_templatelc_for_tfa(task):
    '''
    This is a parallel worker that reforms light curves for TFA.

    task[0] = lcfile
    task[1] = lcformat
    task[2] = lcformatdir
    task[3] = timecol
    task[4] = magcol
    task[5] = errcol
    task[6] = timebase
    task[7] = interpolate_type
    task[8] = sigclip

    '''

    try:

        (lcfile, lcformat, lcformatdir,
         tcol, mcol, ecol,
         timebase, interpolate_type, sigclip) = task

        try:
            formatinfo = get_lcformat(lcformat,
                                      use_lcformat_dir=lcformatdir)
            if formatinfo:
                (dfileglob, readerfunc,
                 dtimecols, dmagcols, derrcols,
                 magsarefluxes, normfunc) = formatinfo
            else:
                LOGERROR("can't figure out the light curve format")
                return None
        except Exception as e:
            LOGEXCEPTION("can't figure out the light curve format")
            return None

        # get the LC into a dict
        lcdict = readerfunc(lcfile)

        # this should handle lists/tuples being returned by readerfunc
        # we assume that the first element is the actual lcdict
        # FIXME: figure out how to not need this assumption
        if ( (isinstance(lcdict, (list, tuple))) and
             (isinstance(lcdict[0], dict)) ):
            lcdict = lcdict[0]

        outdict = {}

        # dereference the columns and get them from the lcdict
        if '.' in tcol:
            tcolget = tcol.split('.')
        else:
            tcolget = [tcol]
        times = _dict_get(lcdict, tcolget)

        if '.' in mcol:
            mcolget = mcol.split('.')
        else:
            mcolget = [mcol]
        mags = _dict_get(lcdict, mcolget)

        if '.' in ecol:
            ecolget = ecol.split('.')
        else:
            ecolget = [ecol]
        errs = _dict_get(lcdict, ecolget)

        # normalize here if not using special normalization
        if normfunc is None:
            ntimes, nmags = normalize_magseries(
                times, mags,
                magsarefluxes=magsarefluxes
            )

        times, mags, errs = ntimes, nmags, errs

        #
        # now we'll do: 1. sigclip, 2. reform to timebase, 3. renorm to zero
        #

        # 1. sigclip as requested
        stimes, smags, serrs = sigclip_magseries(times,
                                                 mags,
                                                 errs,
                                                 sigclip=sigclip)

        # 2. now, we'll renorm to the timebase
        mags_interpolator = spi.interp1d(stimes, smags,
                                         kind=interpolate_type,
                                         fill_value='extrapolate')
        errs_interpolator = spi.interp1d(stimes, serrs,
                                         kind=interpolate_type,
                                         fill_value='extrapolate')

        interpolated_mags = mags_interpolator(timebase)
        interpolated_errs = errs_interpolator(timebase)

        # 3. renorm to zero
        magmedian = np.median(interpolated_mags)

        renormed_mags = interpolated_mags - magmedian

        # update the dict
        outdict = {'mags':renormed_mags,
                   'errs':interpolated_errs,
                   'origmags':interpolated_mags}

        #
        # done with this magcol
        #
        return outdict

    except Exception as e:

        LOGEXCEPTION('reform LC task failed: %s' % repr(task))
        return None

This is a parallel worker that reforms light curves for TFA .


def parallel_tfa_lclist(lclist,
                        templateinfo,
                        timecols=None,
                        magcols=None,
                        errcols=None,
                        lcformat='hat-sql',
                        lcformatdir=None,
                        interp='nearest',
                        sigclip=5.0,
                        mintemplatedist_arcmin=10.0,
                        nworkers=NCPUS,
                        maxworkertasks=1000):
    '''This applies TFA in parallel to all LCs in the given list of file names.

    Parameters
    ----------

    lclist : str
        This is a list of light curve files to apply TFA correction to.

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    timecols : list of str or None
        The timecol keys to use from the lcdict in applying TFA corrections.

    magcols : list of str or None
        The magcol keys to use from the lcdict in applying TFA corrections.

    errcols : list of str or None
        The errcol keys to use from the lcdict in applying TFA corrections.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming the light curves to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to the light curves before running TFA
        on it.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    nworkers : int
        The number of parallel workers to launch

    maxworkertasks : int
        The maximum number of tasks per worker allowed before it's replaced by a
        fresh one.

    Returns
    -------

    dict
        Contains the input file names and output TFA light curve filenames per
        input file organized by each `magcol` in `magcols`.

    '''

    # open the templateinfo first
    if isinstance(templateinfo,str) and os.path.exists(templateinfo):
        with open(templateinfo,'rb') as infd:
            templateinfo = pickle.load(infd)

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # override the default timecols, magcols, and errcols
    # using the ones provided to the function
    # we'll get the defaults from the templateinfo object
    if timecols is None:
        timecols = templateinfo['timecols']
    if magcols is None:
        magcols = templateinfo['magcols']
    if errcols is None:
        errcols = templateinfo['errcols']

    outdict = {}

    # run by magcol
    for t, m, e in zip(timecols, magcols, errcols):

        tasks = [(x, t, m, e, templateinfo,
                  lcformat, lcformatdir,
                  interp, sigclip) for
                 x in lclist]

        pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)
        results = pool.map(_parallel_tfa_worker, tasks)
        pool.close()
        pool.join()

        outdict[m] = results

    return outdict

This applies TFA in parallel to all LCs in the given list of file names .


def parallel_tfa_lcdir(lcdir,
                       templateinfo,
                       lcfileglob=None,
                       timecols=None,
                       magcols=None,
                       errcols=None,
                       lcformat='hat-sql',
                       lcformatdir=None,
                       interp='nearest',
                       sigclip=5.0,
                       mintemplatedist_arcmin=10.0,
                       nworkers=NCPUS,
                       maxworkertasks=1000):
    '''This applies TFA in parallel to all LCs in a directory.

    Parameters
    ----------

    lcdir : str
        This is the directory containing the light curve files to process..

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    lcfileglob : str or None
        The UNIX file glob to use when searching for light curve files in
        `lcdir`. If None, the default file glob associated with registered LC
        format provided is used.

    timecols : list of str or None
        The timecol keys to use from the lcdict in applying TFA corrections.

    magcols : list of str or None
        The magcol keys to use from the lcdict in applying TFA corrections.

    errcols : list of str or None
        The errcol keys to use from the lcdict in applying TFA corrections.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming the light curves to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to the light curves before running TFA
        on it.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    nworkers : int
        The number of parallel workers to launch

    maxworkertasks : int
        The maximum number of tasks per worker allowed before it's replaced by a
        fresh one.

    Returns
    -------

    dict
        Contains the input file names and output TFA light curve filenames per
        input file organized by each `magcol` in `magcols`.

    '''

    # open the templateinfo first
    if isinstance(templateinfo,str) and os.path.exists(templateinfo):
        with open(templateinfo,'rb') as infd:
            templateinfo = pickle.load(infd)

    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # find all the files matching the lcglob in lcdir
    if lcfileglob is None:
        lcfileglob = dfileglob

    lclist = sorted(glob.glob(os.path.join(lcdir, lcfileglob)))

    return parallel_tfa_lclist(
        lclist,
        templateinfo,
        timecols=timecols,
        magcols=magcols,
        errcols=errcols,
        lcformat=lcformat,
        lcformatdir=None,
        interp=interp,
        sigclip=sigclip,
        mintemplatedist_arcmin=mintemplatedist_arcmin,
        nworkers=nworkers,
        maxworkertasks=maxworkertasks
    )

This applies TFA in parallel to all LCs in a directory .


def _read_pklc(lcfile):
    '''
    This just reads a light curve pickle file.

    Parameters
    ----------

    lcfile : str
        The file name of the pickle to open.

    Returns
    -------

    dict
        This returns an lcdict.

    '''

    if lcfile.endswith('.gz'):

        try:
            with gzip.open(lcfile,'rb') as infd:
                lcdict = pickle.load(infd)
        except UnicodeDecodeError:
            with gzip.open(lcfile,'rb') as infd:
                lcdict = pickle.load(infd, encoding='latin1')

    else:

        try:
            with open(lcfile,'rb') as infd:
                lcdict = pickle.load(infd)
        except UnicodeDecodeError:
            with open(lcfile,'rb') as infd:
                lcdict = pickle.load(infd, encoding='latin1')

    return lcdict

def _read_pklc ( lcfile ) : if lcfile . endswith ( '.gz' ) : try : with gzip . open ( lcfile , 'rb' ) as infd : lcdict = pickle . load ( infd ) except UnicodeDecodeError : with gzip . open ( lcfile , 'rb' ) as infd : lcdict = pickle . load ( infd , encoding = 'latin1' ) else : try : with open ( lcfile , 'rb' ) as infd : lcdict = pickle . load ( infd ) except UnicodeDecodeError : with open ( lcfile , 'rb' ) as infd : lcdict = pickle . load ( infd , encoding = 'latin1' ) return lcdict

This just reads a light curve pickle file .


def _check_extmodule(module, formatkey):
    '''This imports the module specified.

    Used to dynamically import Python modules that are needed to support LC
    formats not natively supported by astrobase.

    Parameters
    ----------

    module : str
        This is either:

        - a Python module import path, e.g. 'astrobase.lcproc.catalogs' or
        - a path to a Python file, e.g. '/astrobase/hatsurveys/hatlc.py'

        that contains the Python module that contains functions used to open
        (and optionally normalize) a custom LC format that's not natively
        supported by astrobase.

    formatkey : str
        A str used as the unique ID of this LC format for all lcproc functions
        and can be used to look it up later and import the correct functions
        needed to support it for lcproc operations. For example, we use
        'kep-fits' as a the specifier for Kepler FITS light curves, which can be
        read by the `astrobase.astrokep.read_kepler_fitslc` function as
        specified by the `<astrobase install path>/data/lcformats/kep-fits.json`
        LC format specification JSON.

    Returns
    -------

    Python module
        This returns a Python module if it's able to successfully import it.

    '''

    try:

        if os.path.exists(module):

            sys.path.append(os.path.dirname(module))
            importedok = importlib.import_module(
                os.path.basename(module.replace('.py',''))
            )

        else:
            importedok = importlib.import_module(module)

    except Exception as e:

        LOGEXCEPTION('could not import the module: %s for LC format: %s. '
                     'check the file path or fully qualified module name?'
                     % (module, formatkey))
        importedok = False

    return importedok


def register_lcformat(formatkey,
                      fileglob,
                      timecols,
                      magcols,
                      errcols,
                      readerfunc_module,
                      readerfunc,
                      readerfunc_kwargs=None,
                      normfunc_module=None,
                      normfunc=None,
                      normfunc_kwargs=None,
                      magsarefluxes=False,
                      overwrite_existing=False,
                      lcformat_dir='~/.astrobase/lcformat-jsons'):
    '''This adds a new LC format to the astrobase LC format registry.

    Allows handling of custom format light curves for astrobase lcproc
    drivers. Once the format is successfully registered, light curves should
    work transparently with all of the functions in this module, by simply
    calling them with the `formatkey` in the `lcformat` keyword argument.

    LC format specifications are generated as JSON files. astrobase comes with
    several of these in `<astrobase install path>/data/lcformats`. LC formats
    you add by using this function will have their specifiers written to the
    `~/.astrobase/lcformat-jsons` directory in your home directory.

    Parameters
    ----------

    formatkey : str
        A str used as the unique ID of this LC format for all lcproc functions
        and can be used to look it up later and import the correct functions
        needed to support it for lcproc operations. For example, we use
        'kep-fits' as a the specifier for Kepler FITS light curves, which can be
        read by the `astrobase.astrokep.read_kepler_fitslc` function as
        specified by the `<astrobase install path>/data/lcformats/kep-fits.json`
        LC format specification JSON produced by `register_lcformat`.

    fileglob : str
        The default UNIX fileglob to use to search for light curve files in this
        LC format. This is a string like '*-whatever-???-*.*??-.lc'.

    timecols,magcols,errcols : list of str
        These are all lists of strings indicating which keys in the lcdict
        produced by your `lcreader_func` that will be extracted and used by
        lcproc functions for processing. The lists must all have the same
        dimensions, e.g. if timecols = ['timecol1','timecol2'], then magcols
        must be something like ['magcol1','magcol2'] and errcols must be
        something like ['errcol1', 'errcol2']. This allows you to process
        multiple apertures or multiple types of measurements in one go.

        Each element in these lists can be a simple key, e.g. 'time' (which
        would correspond to lcdict['time']), or a composite key,
        e.g. 'aperture1.times.rjd' (which would correspond to
        lcdict['aperture1']['times']['rjd']). See the examples in the lcformat
        specification JSON files in `<astrobase install path>/data/lcformats`.

    readerfunc_module : str
        This is either:

        - a Python module import path, e.g. 'astrobase.lcproc.catalogs' or
        - a path to a Python file, e.g. '/astrobase/hatsurveys/hatlc.py'

        that contains the Python module that contains functions used to open
        (and optionally normalize) a custom LC format that's not natively
        supported by astrobase.

    readerfunc : str
        This is the function name in `readerfunc_module` to use to read light
        curves in the custom format. This MUST always return a dictionary (the
        'lcdict') with the following signature (the keys listed below are
        required, but others are allowed)::

            {'objectid': this object's identifier as a string,
             'objectinfo':{'ra': this object's right ascension in decimal deg,
                           'decl': this object's declination in decimal deg,
                           'ndet': the number of observations in this LC,
                           'objectid': the object ID again for legacy reasons},
             ...other time columns, mag columns go in as their own keys}

    normfunc_kwargs : dict or None
        This is a dictionary containing any kwargs to pass through to
        the light curve norm function.

    normfunc_module : str or None
        This is either:

        - a Python module import path, e.g. 'astrobase.lcproc.catalogs' or
        - a path to a Python file, e.g. '/astrobase/hatsurveys/hatlc.py'
        - None, in which case we'll use default normalization

        that contains the Python module that contains functions used to
        normalize a custom LC format that's not natively supported by astrobase.

    normfunc : str or None
        This is the function name in `normfunc_module` to use to normalize light
        curves in the custom format. If None, the default normalization method
        used by lcproc is to find gaps in the time-series, normalize
        measurements grouped by these gaps to zero, then normalize the entire
        magnitude time series to global time series median using the
        `astrobase.lcmath.normalize_magseries` function.

        If this is provided, the normalization function should take and return
        an lcdict of the same form as that produced by `readerfunc` above. For
        an example of a specific normalization function, see
        `normalize_lcdict_by_inst` in the `astrobase.hatsurveys.hatlc` module.

    normfunc_kwargs : dict or None
        This is a dictionary containing any kwargs to pass through to
        the light curve normalization function.

    magsarefluxes : bool
        If this is True, then all lcproc functions will treat the measurement
        columns in the lcdict produced by your `readerfunc` as flux instead of
        mags, so things like default normalization and sigma-clipping will be
        done correctly. If this is False, magnitudes will be treated as
        magnitudes.

    overwrite_existing : bool
        If this is True, this function will overwrite any existing LC format
        specification JSON with the same name as that provided in the
        `formatkey` arg. This can be used to update LC format specifications
        while keeping the `formatkey` the same.

    lcformat_dir : str
        This specifies the directory where the the LC format specification JSON
        produced by this function will be written. By default, this goes to the
        `.astrobase/lcformat-jsons` directory in your home directory.

    Returns
    -------

    str
        Returns the file path to the generated LC format specification JSON
        file.

    '''

    LOGINFO('adding %s to LC format registry...' % formatkey)

    # search for the lcformat_dir and create it if it doesn't exist
    lcformat_dpath = os.path.abspath(
        os.path.expanduser(lcformat_dir)
    )
    if not os.path.exists(lcformat_dpath):
        os.makedirs(lcformat_dpath)

    lcformat_jsonpath = os.path.join(lcformat_dpath,'%s.json' % formatkey)

    if os.path.exists(lcformat_jsonpath) and not overwrite_existing:
        LOGERROR('There is an existing lcformat JSON: %s '
                 'for this formatkey: %s and '
                 'overwrite_existing = False, skipping...'
                 % (lcformat_jsonpath, formatkey))
        return None

    # see if we can import the reader module
    readermodule = _check_extmodule(readerfunc_module, formatkey)

    if not readermodule:
        LOGERROR("could not import the required "
                 "module: %s to read %s light curves" %
                 (readerfunc_module, formatkey))
        return None

    # then, get the function we need to read the light curve
    try:
        getattr(readermodule, readerfunc)
        readerfunc_in = readerfunc
    except AttributeError:
        LOGEXCEPTION('Could not get the specified reader '
                     'function: %s for lcformat: %s '
                     'from module: %s'
                     % (formatkey, readerfunc_module, readerfunc))
        raise

    # see if we can import the normalization module
    if normfunc_module:
        normmodule = _check_extmodule(normfunc_module, formatkey)
        if not normmodule:
            LOGERROR("could not import the required "
                     "module: %s to normalize %s light curves" %
                     (normfunc_module, formatkey))
            return None

    else:
        normmodule = None

    # finally, get the function we need to normalize the light curve
    if normfunc_module and normfunc:
        try:
            getattr(normmodule, normfunc)
            normfunc_in = normfunc
        except AttributeError:
            LOGEXCEPTION('Could not get the specified norm '
                         'function: %s for lcformat: %s '
                         'from module: %s'
                         % (normfunc, formatkey, normfunc_module))
            raise

    else:
        normfunc_in = None

    # if we made it to here, then everything's good. generate the JSON
    # structure
    formatdict = {'fileglob':fileglob,
                  'timecols':timecols,
                  'magcols':magcols,
                  'errcols':errcols,
                  'magsarefluxes':magsarefluxes,
                  'lcreader_module':readerfunc_module,
                  'lcreader_func':readerfunc_in,
                  'lcreader_kwargs':readerfunc_kwargs,
                  'lcnorm_module':normfunc_module,
                  'lcnorm_func':normfunc_in,
                  'lcnorm_kwargs':normfunc_kwargs}

    # write this to the lcformat directory
    with open(lcformat_jsonpath,'w') as outfd:
        json.dump(formatdict, outfd, indent=4)

    return lcformat_jsonpath

This adds a new LC format to the astrobase LC format registry .


def ec2_ssh(ip_address,
            keypem_file,
            username='ec2-user',
            raiseonfail=False):
    """This opens an SSH connection to the EC2 instance at `ip_address`.

    Parameters
    ----------

    ip_address : str
        IP address of the AWS EC2 instance to connect to.

    keypem_file : str
        The path to the keypair PEM file generated by AWS to allow SSH
        connections.

    username : str
        The username to use to login to the EC2 instance.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    paramiko.SSHClient
        This has all the usual `paramiko` functionality:

        - Use `SSHClient.exec_command(command, environment=None)` to exec a
          shell command.

        - Use `SSHClient.open_sftp()` to get a `SFTPClient` for the server. Then
          call SFTPClient.get() and .put() to copy files from and to the server.

    """

    c = paramiko.client.SSHClient()
    c.load_system_host_keys()
    c.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)

    # load the private key from the AWS keypair pem
    privatekey = paramiko.RSAKey.from_private_key_file(keypem_file)

    # connect to the server
    try:

        c.connect(ip_address,
                  pkey=privatekey,
                  username='ec2-user')

        return c

    except Exception as e:
        LOGEXCEPTION('could not connect to EC2 instance at %s '
                     'using keyfile: %s and user: %s' %
                     (ip_address, keypem_file, username))
        if raiseonfail:
            raise

        return None

This opens an SSH connection to the EC2 instance at ip_address .


def s3_get_file(bucket,
                filename,
                local_file,
                altexts=None,
                client=None,
                raiseonfail=False):

    """This gets a file from an S3 bucket.

    Parameters
    ----------

    bucket : str
        The AWS S3 bucket name.

    filename : str
        The full filename of the file to get from the bucket

    local_file : str
        Path to where the downloaded file will be stored.

    altexts : None or list of str
        If not None, this is a list of alternate extensions to try for the file
        other than the one provided in `filename`. For example, to get anything
        that's an .sqlite where .sqlite.gz is expected, use altexts=[''] to
        strip the .gz.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str
        Path to the downloaded filename or None if the download was
        unsuccessful.

    """

    if not client:
        client = boto3.client('s3')

    try:

        client.download_file(bucket, filename, local_file)
        return local_file

    except Exception as e:

        if altexts is not None:

            for alt_extension in altexts:

                split_ext = os.path.splitext(filename)
                check_file = split_ext[0] + alt_extension
                try:
                    client.download_file(
                        bucket,
                        check_file,
                        local_file.replace(split_ext[-1],
                                           alt_extension)
                    )
                    return local_file.replace(split_ext[-1],
                                              alt_extension)
                except Exception as e:
                    pass

        else:

            LOGEXCEPTION('could not download s3://%s/%s' % (bucket, filename))

            if raiseonfail:
                raise

            return None

def
s3_get_file
(
bucket
,
filename
,
local_file
,
altexts
=
,
client
=
,
raiseonfail
=
False
)
:
if
not
client
:
client
=
boto3
.
client
(
's3'
)
try
:
client
.
download_file
(
bucket
,
filename
,
local_file
)
return
local_file
except
Exception
as
e
:
if
altexts
is
not
:
for
alt_extension
in
altexts
:
split_ext
=
os
.
path
.
splitext
(
filename
)
check_file
=
split_ext
[
0
]
+
alt_extension
try
:
client
.
download_file
(
bucket
,
check_file
,
local_file
.
replace
(
split_ext
[
1
]
,
alt_extension
)
)
return
local_file
.
replace
(
split_ext
[
1
]
,
alt_extension
)
except
Exception
as
e
:
pass
else
:
LOGEXCEPTION
(
'could not download s3://%s/%s'
%
(
bucket
,
filename
)
)
if
raiseonfail
:
raise
return


def s3_put_file(local_file, bucket, client=None, raiseonfail=False):
    """This uploads a file to S3.

    Parameters
    ----------

    local_file : str
        Path to the file to upload to S3.

    bucket : str
        The AWS S3 bucket to upload the file to.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        If the file upload is successful, returns the s3:// URL of the uploaded
        file. If it failed, will return None.

    """

    if not client:
        client = boto3.client('s3')

    try:
        client.upload_file(local_file, bucket, os.path.basename(local_file))
        return 's3://%s/%s' % (bucket, os.path.basename(local_file))
    except Exception as e:
        LOGEXCEPTION('could not upload %s to bucket: %s' % (local_file,
                                                            bucket))

        if raiseonfail:
            raise

        return None

def
s3_put_file
(
local_file
,
bucket
,
client
=
,
raiseonfail
=
False
)
:
if
not
client
:
client
=
boto3
.
client
(
's3'
)
try
:
client
.
upload_file
(
local_file
,
bucket
,
os
.
path
.
basename
(
local_file
)
)
return
's3://%s/%s'
%
(
bucket
,
os
.
path
.
basename
(
local_file
)
)
except
Exception
as
e
:
LOGEXCEPTION
(
'could not upload %s to bucket: %s'
%
(
local_file
,
bucket
)
)
if
raiseonfail
:
raise
return


def s3_delete_file(bucket, filename, client=None, raiseonfail=False):
    """This deletes a file from S3.

    Parameters
    ----------

    bucket : str
        The AWS S3 bucket to delete the file from.

    filename : str
        The full file name of the file to delete, including any prefixes.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        If the file was successfully deleted, will return the delete-marker
        (https://docs.aws.amazon.com/AmazonS3/latest/dev/DeleteMarker.html). If
        it wasn't, returns None

    """

    if not client:
        client = boto3.client('s3')

    try:
        resp = client.delete_object(Bucket=bucket, Key=filename)
        if not resp:
            LOGERROR('could not delete file %s from bucket %s' % (filename,
                                                                  bucket))
        else:
            return resp['DeleteMarker']
    except Exception as e:
        LOGEXCEPTION('could not delete file %s from bucket %s' % (filename,
                                                                  bucket))
        if raiseonfail:
            raise

        return None


def sqs_create_queue(queue_name, options=None, client=None):
    """
    This creates an SQS queue.

    Parameters
    ----------

    queue_name : str
        The name of the queue to create.

    options : dict or None
        A dict of options indicate extra attributes the queue should have.
        See the SQS docs for details. If None, no custom attributes will be
        attached to the queue.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    Returns
    -------

    dict
        This returns a dict of the form::

            {'url': SQS URL of the queue,
             'name': name of the queue}

    """

    if not client:
        client = boto3.client('sqs')

    try:

        if isinstance(options, dict):
            resp = client.create_queue(QueueName=queue_name, Attributes=options)
        else:
            resp = client.create_queue(QueueName=queue_name)

        if resp is not None:
            return {'url':resp['QueueUrl'],
                    'name':queue_name}
        else:
            LOGERROR('could not create the specified queue: %s with options: %s'
                     % (queue_name, options))
            return None

    except Exception as e:
        LOGEXCEPTION('could not create the specified queue: %s with options: %s'
                     % (queue_name, options))
        return None


def sqs_delete_queue(queue_url, client=None):
    """This deletes an SQS queue given its URL

    Parameters
    ----------

    queue_url : str
        The SQS URL of the queue to delete.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    Returns
    -------

    bool
        True if the queue was deleted successfully. False otherwise.

    """

    if not client:
        client = boto3.client('sqs')

    try:

        client.delete_queue(QueueUrl=queue_url)
        return True

    except Exception as e:
        LOGEXCEPTION('could not delete the specified queue: %s'
                     % (queue_url,))
        return False


def sqs_put_item(queue_url,
                 item,
                 delay_seconds=0,
                 client=None,
                 raiseonfail=False):
    """This pushes a dict serialized to JSON to the specified SQS queue.

    Parameters
    ----------

    queue_url : str
        The SQS URL of the queue to push the object to.

    item : dict
        The dict passed in here will be serialized to JSON.

    delay_seconds : int
        The amount of time in seconds the pushed item will be held before going
        'live' and being visible to all queue consumers.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    boto3.Response or None
        If the item was successfully put on the queue, will return the response
        from the service. If it wasn't, will return None.

    """

    if not client:
        client = boto3.client('sqs')

    try:

        json_msg = json.dumps(item)

        resp = client.send_message(
            QueueUrl=queue_url,
            MessageBody=json_msg,
            DelaySeconds=delay_seconds,
        )
        if not resp:
            LOGERROR('could not send item to queue: %s' % queue_url)
            return None
        else:
            return resp

    except Exception as e:

        LOGEXCEPTION('could not send item to queue: %s' % queue_url)

        if raiseonfail:
            raise

        return None

def
sqs_put_item
(
queue_url
,
item
,
delay_seconds
=
0
,
client
=
,
raiseonfail
=
False
)
:
if
not
client
:
client
=
boto3
.
client
(
'sqs'
)
try
:
json_msg
=
json
.
dumps
(
item
)
resp
=
client
.
send_message
(
QueueUrl
=
queue_url
,
MessageBody
=
json_msg
,
DelaySeconds
=
delay_seconds
,
)
if
not
resp
:
LOGERROR
(
'could not send item to queue: %s'
%
queue_url
)
return
else
:
return
resp
except
Exception
as
e
:
LOGEXCEPTION
(
'could not send item to queue: %s'
%
queue_url
)
if
raiseonfail
:
raise
return

This pushes a dict serialized to JSON to the specified SQS queue .


def sqs_get_item(queue_url,
                 max_items=1,
                 wait_time_seconds=5,
                 client=None,
                 raiseonfail=False):
    """This gets a single item from the SQS queue.

    The `queue_url` is composed of some internal SQS junk plus a
    `queue_name`. For our purposes (`lcproc_aws.py`), the queue name will be
    something like::

        lcproc_queue_<action>

    where action is one of::

        runcp
        runpf

    The item is always a JSON object::

        {'target': S3 bucket address of the file to process,
         'action': the action to perform on the file ('runpf', 'runcp', etc.)
         'args': the action's args as a tuple (not including filename, which is
                 generated randomly as a temporary local file),
         'kwargs': the action's kwargs as a dict,
         'outbucket: S3 bucket to write the result to,
         'outqueue': SQS queue to write the processed item's info to (optional)}

    The action MUST match the <action> in the queue name for this item to be
    processed.

    Parameters
    ----------

    queue_url : str
        The SQS URL of the queue to get messages from.

    max_items : int
        The number of items to pull from the queue in this request.

    wait_time_seconds : int
        This specifies how long the function should block until a message is
        received on the queue. If the timeout expires, an empty list will be
        returned. If the timeout doesn't expire, the function will return a list
        of items received (up to `max_items`).

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    list of dicts or None
        For each item pulled from the queue in this request (up to `max_items`),
        a dict will be deserialized from the retrieved JSON, containing the
        message items and various metadata. The most important item of the
        metadata is the `receipt_handle`, which can be used to acknowledge
        receipt of all items in this request (see `sqs_delete_item` below).

        If the queue pull fails outright, returns None. If no messages are
        available for this queue pull, returns an empty list.

    """

    if not client:
        client = boto3.client('sqs')

    try:

        resp = client.receive_message(
            QueueUrl=queue_url,
            AttributeNames=['All'],
            MaxNumberOfMessages=max_items,
            WaitTimeSeconds=wait_time_seconds
        )

        if not resp:
            LOGERROR('could not receive messages from queue: %s' %
                     queue_url)

        else:

            messages = []

            for msg in resp.get('Messages',[]):

                try:
                    messages.append({
                        'id':msg['MessageId'],
                        'receipt_handle':msg['ReceiptHandle'],
                        'md5':msg['MD5OfBody'],
                        'attributes':msg['Attributes'],
                        'item':json.loads(msg['Body']),
                    })
                except Exception as e:
                    LOGEXCEPTION(
                        'could not deserialize message ID: %s, body: %s' %
                        (msg['MessageId'], msg['Body'])
                    )
                    continue

            return messages

    except Exception as e:
        LOGEXCEPTION('could not get items from queue: %s' % queue_url)

        if raiseonfail:
            raise

        return None

This gets a single item from the SQS queue .


def sqs_delete_item(queue_url,
                    receipt_handle,
                    client=None,
                    raiseonfail=False):
    """This deletes a message from the queue, effectively acknowledging its
    receipt.

    Call this only when all messages retrieved from the queue have been
    processed, since this will prevent redelivery of these messages to other
    queue workers pulling fromn the same queue channel.

    Parameters
    ----------

    queue_url : str
        The SQS URL of the queue where we got the messages from. This should be
        the same queue used to retrieve the messages in `sqs_get_item`.

    receipt_handle : str
        The receipt handle of the queue message that we're responding to, and
        will acknowledge receipt of. This will be present in each message
        retrieved using `sqs_get_item`.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    Nothing.

    """

    if not client:
        client = boto3.client('sqs')

    try:

        client.delete_message(
            QueueUrl=queue_url,
            ReceiptHandle=receipt_handle
        )

    except Exception as e:

        LOGEXCEPTION(
            'could not delete message with receipt handle: '
            '%s from queue: %s' % (receipt_handle, queue_url)
        )

        if raiseonfail:
            raise

This deletes a message from the queue effectively acknowledging its receipt .


def delete_ec2_nodes(
        instance_id_list,
        client=None
):
    """This deletes EC2 nodes and terminates the instances.

    Parameters
    ----------

    instance_id_list : list of str
        A list of EC2 instance IDs to terminate.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    Returns
    -------

    Nothing.

    """

    if not client:
        client = boto3.client('ec2')

    resp = client.terminate_instances(
        InstanceIds=instance_id_list
    )

    return resp

def delete_ec2_nodes ( instance_id_list , client = None ) : if not client : client = boto3 . client ( 'ec2' ) resp = client . terminate_instances ( InstanceIds = instance_id_list ) return resp

This deletes EC2 nodes and terminates the instances .


def delete_spot_fleet_cluster(
        spot_fleet_reqid,
        client=None,
):
    """
    This deletes a spot-fleet cluster.

    Parameters
    ----------

    spot_fleet_reqid : str
        The fleet request ID returned by `make_spot_fleet_cluster`.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    Returns
    -------

    Nothing.

    """

    if not client:
        client = boto3.client('ec2')

    resp = client.cancel_spot_fleet_requests(
        SpotFleetRequestIds=[spot_fleet_reqid],
        TerminateInstances=True
    )

    return resp

def delete_spot_fleet_cluster ( spot_fleet_reqid , client = None , ) : if not client : client = boto3 . client ( 'ec2' ) resp = client . cancel_spot_fleet_requests ( SpotFleetRequestIds = [ spot_fleet_reqid ] , TerminateInstances = True ) return resp


def gcs_put_file(local_file,
                 bucketname,
                 service_account_json=None,
                 client=None,
                 raiseonfail=False):
    """This puts a single file into a Google Cloud Storage bucket.

    Parameters
    ----------

    local_file : str
        Path to the file to upload to GCS.

    bucket : str
        The GCS bucket to upload the file to.

    service_account_json : str
        Path to a downloaded GCS credentials JSON file.

    client : google.cloud.storage.Client instance
        The instance of the Client to use to perform the download operation. If
        this is None, a new Client will be used. If this is None and
        `service_account_json` points to a downloaded JSON file with GCS
        credentials, a new Client with the provided credentials will be used. If
        this is not None, the existing Client instance will be used.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        If the file upload is successful, returns the gs:// URL of the uploaded
        file. If it failed, will return None.

    """

    if not client:

        if (service_account_json is not None and
            os.path.exists(service_account_json)):
            client = storage.Client.from_service_account_json(
                service_account_json
            )
        else:
            client = storage.Client()

    try:

        bucket = client.get_bucket(bucketname)
        remote_blob = bucket.blob(local_file)
        remote_blob.upload_from_filename(local_file)
        return 'gs://%s/%s' % (bucketname, local_file.lstrip('/'))

    except Exception as e:

        LOGEXCEPTION('could not upload %s to bucket %s' % (local_file,
                                                           bucket))

        if raiseonfail:
            raise

        return None

def
gcs_put_file
(
local_file
,
bucketname
,
service_account_json
=
,
client
=
,
raiseonfail
=
False
)
:
if
not
client
:
if
(
service_account_json
is
not
and
os
.
path
.
exists
(
service_account_json
)
)
:
client
=
storage
.
Client
.
from_service_account_json
(
service_account_json
)
else
:
client
=
storage
.
Client
(
)
try
:
bucket
=
client
.
get_bucket
(
bucketname
)
remote_blob
=
bucket
.
blob
(
local_file
)
remote_blob
.
upload_from_filename
(
local_file
)
return
'gs://%s/%s'
%
(
bucketname
,
local_file
.
lstrip
(
'/'
)
)
except
Exception
as
e
:
LOGEXCEPTION
(
'could not upload %s to bucket %s'
%
(
local_file
,
bucket
)
)
if
raiseonfail
:
raise
return

This puts a single file into a Google Cloud Storage bucket .


def read_fakelc(fakelcfile):
    '''
    This just reads a pickled fake LC.

    Parameters
    ----------

    fakelcfile : str
        The fake LC file to read.

    Returns
    -------

    dict
        This returns an lcdict.

    '''

    try:
        with open(fakelcfile,'rb') as infd:
            lcdict = pickle.load(infd)
    except UnicodeDecodeError:
        with open(fakelcfile,'rb') as infd:
            lcdict = pickle.load(infd, encoding='latin1')

    return lcdict

def read_fakelc ( fakelcfile ) : try : with open ( fakelcfile , 'rb' ) as infd : lcdict = pickle . load ( infd ) except UnicodeDecodeError : with open ( fakelcfile , 'rb' ) as infd : lcdict = pickle . load ( infd , encoding = 'latin1' ) return lcdict


def get_varfeatures(simbasedir,
                    mindet=1000,
                    nworkers=None):
    '''This runs `lcproc.lcvfeatures.parallel_varfeatures` on fake LCs in
    `simbasedir`.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    mindet : int
        The minimum number of detections needed to accept an LC and process it.

    nworkers : int or None
        The number of parallel workers to use when extracting variability
        features from the input light curves.

    Returns
    -------

    str
        The path to the `varfeatures` pickle created after running the
        `lcproc.lcvfeatures.parallel_varfeatures` function.

    '''

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    lcfpaths = siminfo['lcfpath']
    varfeaturedir = os.path.join(simbasedir,'varfeatures')

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # register the fakelc pklc as a custom lcproc format
    # now we should be able to use all lcproc functions correctly
    fakelc_formatkey = 'fake-%s' % siminfo['lcformat']
    lcproc.register_lcformat(
        fakelc_formatkey,
        '*-fakelc.pkl',
        timecols,
        magcols,
        errcols,
        'astrobase.lcproc',
        '_read_pklc',
        magsarefluxes=siminfo['magsarefluxes']
    )

    # now we can use lcproc.parallel_varfeatures directly
    varinfo = lcvfeatures.parallel_varfeatures(lcfpaths,
                                               varfeaturedir,
                                               lcformat=fakelc_formatkey,
                                               mindet=mindet,
                                               nworkers=nworkers)

    with open(os.path.join(simbasedir,'fakelc-varfeatures.pkl'),'wb') as outfd:
        pickle.dump(varinfo, outfd, pickle.HIGHEST_PROTOCOL)

    return os.path.join(simbasedir,'fakelc-varfeatures.pkl')

This runs lcproc . lcvfeatures . parallel_varfeatures on fake LCs in simbasedir .


def precision(ntp, nfp):
    '''
    This calculates precision.

    https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------

    ntp : int
        The number of true positives.

    nfp : int
        The number of false positives.

    Returns
    -------

    float
        The precision calculated using `ntp/(ntp + nfp)`.

    '''

    if (ntp+nfp) > 0:
        return ntp/(ntp+nfp)
    else:
        return np.nan

def precision ( ntp , nfp ) : if ( ntp + nfp ) > 0 : return ntp / ( ntp + nfp ) else : return np . nan


def recall(ntp, nfn):
    '''
    This calculates recall.

    https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------

    ntp : int
        The number of true positives.

    nfn : int
        The number of false negatives.

    Returns
    -------

    float
        The precision calculated using `ntp/(ntp + nfn)`.

    '''

    if (ntp+nfn) > 0:
        return ntp/(ntp+nfn)
    else:
        return np.nan

def recall ( ntp , nfn ) : if ( ntp + nfn ) > 0 : return ntp / ( ntp + nfn ) else : return np . nan


def matthews_correl_coeff(ntp, ntn, nfp, nfn):
    '''
    This calculates the Matthews correlation coefficent.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    Parameters
    ----------

    ntp : int
        The number of true positives.

    ntn : int
        The number of true negatives

    nfp : int
        The number of false positives.

    nfn : int
        The number of false negatives.

    Returns
    -------

    float
        The Matthews correlation coefficient.

    '''

    mcc_top = (ntp*ntn - nfp*nfn)
    mcc_bot = msqrt((ntp + nfp)*(ntp + nfn)*(ntn + nfp)*(ntn + nfn))

    if mcc_bot > 0:
        return mcc_top/mcc_bot
    else:
        return np.nan

def matthews_correl_coeff ( ntp , ntn , nfp , nfn ) : mcc_top = ( ntp * ntn - nfp * nfn ) mcc_bot = msqrt ( ( ntp + nfp ) * ( ntp + nfn ) * ( ntn + nfp ) * ( ntn + nfn ) ) if mcc_bot > 0 : return mcc_top / mcc_bot else : return np . nan

This calculates the Matthews correlation coefficent .


def magbin_varind_gridsearch_worker(task):
    '''
    This is a parallel grid search worker for the function below.

    '''

    simbasedir, gridpoint, magbinmedian = task

    try:
        res = get_recovered_variables_for_magbin(simbasedir,
                                                 magbinmedian,
                                                 stetson_stdev_min=gridpoint[0],
                                                 inveta_stdev_min=gridpoint[1],
                                                 iqr_stdev_min=gridpoint[2],
                                                 statsonly=True)
        return res
    except Exception as e:
        LOGEXCEPTION('failed to get info for %s' % gridpoint)
        return None

def
magbin_varind_gridsearch_worker
(
task
)
:
simbasedir
,
gridpoint
,
magbinmedian
=
task
try
:
res
=
get_recovered_variables_for_magbin
(
simbasedir
,
magbinmedian
,
stetson_stdev_min
=
gridpoint
[
0
]
,
inveta_stdev_min
=
gridpoint
[
1
]
,
iqr_stdev_min
=
gridpoint
[
2
]
,
statsonly
=
True
)
return
res
except
Exception
as
e
:
LOGEXCEPTION
(
'failed to get info for %s'
%
gridpoint
)
return

This is a parallel grid search worker for the function below .


def variable_index_gridsearch_magbin(simbasedir,
                                     stetson_stdev_range=(1.0,20.0),
                                     inveta_stdev_range=(1.0,20.0),
                                     iqr_stdev_range=(1.0,20.0),
                                     ngridpoints=32,
                                     ngridworkers=None):
    '''This runs a variable index grid search per magbin.

    For each magbin, this does a grid search using the stetson and inveta ranges
    provided and tries to optimize the Matthews Correlation Coefficient (best
    value is +1.0), indicating the best possible separation of variables
    vs. nonvariables. The thresholds on these two variable indexes that produce
    the largest coeff for the collection of fake LCs will probably be the ones
    that work best for actual variable classification on the real LCs.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    For each grid-point, calculates the true positives, false positives, true
    negatives, false negatives. Then gets the precision and recall, confusion
    matrix, and the ROC curve for variable vs. nonvariable.

    Once we've identified the best thresholds to use, we can then calculate
    variable object numbers:

    - as a function of magnitude
    - as a function of period
    - as a function of number of detections
    - as a function of amplitude of variability

    Writes everything back to `simbasedir/fakevar-recovery.pkl`. Use the
    plotting function below to make plots for the results.

    Parameters
    ----------

    simbasedir : str
        The directory where the fake LCs are located.

    stetson_stdev_range : sequence of 2 floats
        The min and max values of the Stetson J variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    inveta_stdev_range : sequence of 2 floats
        The min and max values of the 1/eta variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    iqr_stdev_range : sequence of 2 floats
        The min and max values of the IQR variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    ngridpoints : int
        The number of grid points for each variability index grid. Remember that
        this function will be searching in 3D and will require lots of time to
        run if ngridpoints is too large.

        For the default number of grid points and 25000 simulated light curves,
        this takes about 3 days to run on a 40 (effective) core machine with 2 x
        Xeon E5-2650v3 CPUs.

    ngridworkers : int or None
        The number of parallel grid search workers that will be launched.

    Returns
    -------

    dict
        The returned dict contains a list of recovery stats for each magbin and
        each grid point in the variability index grids that were used. This dict
        can be passed to the plotting function below to plot the results.

    '''

    # make the output directory where all the pkls from the variability
    # threshold runs will go
    outdir = os.path.join(simbasedir,'recvar-threshold-pkls')
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # get the magbinmedians to use for the recovery processing
    magbinmedians = siminfo['magrms'][magcols[0]]['binned_sdssr_median']

    # generate the grids for stetson and inveta
    stetson_grid = np.linspace(stetson_stdev_range[0],
                               stetson_stdev_range[1],
                               num=ngridpoints)
    inveta_grid = np.linspace(inveta_stdev_range[0],
                              inveta_stdev_range[1],
                              num=ngridpoints)
    iqr_grid = np.linspace(iqr_stdev_range[0],
                           iqr_stdev_range[1],
                           num=ngridpoints)

    # generate the grid
    stet_inveta_iqr_grid = []
    for stet in stetson_grid:
        for inveta in inveta_grid:
            for iqr in iqr_grid:
                grid_point = [stet, inveta, iqr]
                stet_inveta_iqr_grid.append(grid_point)

    # the output dict
    grid_results = {'stetson_grid':stetson_grid,
                    'inveta_grid':inveta_grid,
                    'iqr_grid':iqr_grid,
                    'stet_inveta_iqr_grid':stet_inveta_iqr_grid,
                    'magbinmedians':magbinmedians,
                    'timecols':timecols,
                    'magcols':magcols,
                    'errcols':errcols,
                    'simbasedir':os.path.abspath(simbasedir),
                    'recovery':[]}

    # set up the pool
    pool = mp.Pool(ngridworkers)

    # run the grid search per magbinmedian
    for magbinmedian in magbinmedians:

        LOGINFO('running stetson J-inveta grid-search '
                'for magbinmedian = %.3f...' % magbinmedian)

        tasks = [(simbasedir, gp, magbinmedian) for gp in stet_inveta_iqr_grid]
        thisbin_results = pool.map(magbin_varind_gridsearch_worker, tasks)
        grid_results['recovery'].append(thisbin_results)

    pool.close()
    pool.join()

    LOGINFO('done.')
    with open(os.path.join(simbasedir,
                           'fakevar-recovery-per-magbin.pkl'),'wb') as outfd:
        pickle.dump(grid_results,outfd,pickle.HIGHEST_PROTOCOL)

    return grid_results

This runs a variable index grid search per magbin .


def run_periodfinding(simbasedir,
                      pfmethods=('gls','pdm','bls'),
                      pfkwargs=({},{},{'startp':1.0,'maxtransitduration':0.3}),
                      getblssnr=False,
                      sigclip=5.0,
                      nperiodworkers=10,
                      ncontrolworkers=4,
                      liststartindex=None,
                      listmaxobjects=None):
    '''This runs periodfinding using several period-finders on a collection of
    fake LCs.

    As a rough benchmark, 25000 fake LCs with 10000--50000 points per LC take
    about 26 days in total to run on an invocation of this function using
    GLS+PDM+BLS and 10 periodworkers and 4 controlworkers (so all 40 'cores') on
    a 2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    pfmethods : sequence of str
        This is used to specify which periodfinders to run. These must be in the
        `lcproc.periodsearch.PFMETHODS` dict.

    pfkwargs : sequence of dict
        This is used to provide optional kwargs to the period-finders.

    getblssnr : bool
        If this is True, will run BLS SNR calculations for each object and
        magcol. This takes a while to run, so it's disabled (False) by default.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    ncontrolworkers : int
        This is the number of parallel period-finding control workers to
        use. Each control worker will launch `nperiodworkers` worker processes.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the fake LCs in `simbasedir`.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input light curves over several sessions or machines.

    Returns
    -------

    str
        The path to the output summary pickle produced by
        `lcproc.periodsearch.parallel_pf`

    '''

    # get the info from the simbasedir
    with open(os.path.join(simbasedir, 'fakelcs-info.pkl'),'rb') as infd:
        siminfo = pickle.load(infd)

    lcfpaths = siminfo['lcfpath']
    pfdir = os.path.join(simbasedir,'periodfinding')

    # get the column defs for the fakelcs
    timecols = siminfo['timecols']
    magcols = siminfo['magcols']
    errcols = siminfo['errcols']

    # register the fakelc pklc as a custom lcproc format
    # now we should be able to use all lcproc functions correctly
    fakelc_formatkey = 'fake-%s' % siminfo['lcformat']
    lcproc.register_lcformat(
        fakelc_formatkey,
        '*-fakelc.pkl',
        timecols,
        magcols,
        errcols,
        'astrobase.lcproc',
        '_read_pklc',
        magsarefluxes=siminfo['magsarefluxes']
    )

    if liststartindex:
        lcfpaths = lcfpaths[liststartindex:]

    if listmaxobjects:
        lcfpaths = lcfpaths[:listmaxobjects]

    pfinfo = periodsearch.parallel_pf(lcfpaths,
                                      pfdir,
                                      lcformat=fakelc_formatkey,
                                      pfmethods=pfmethods,
                                      pfkwargs=pfkwargs,
                                      getblssnr=getblssnr,
                                      sigclip=sigclip,
                                      nperiodworkers=nperiodworkers,
                                       ncontrolworkers=ncontrolworkers)

    with open(os.path.join(simbasedir,
                           'fakelc-periodsearch.pkl'),'wb') as outfd:
        pickle.dump(pfinfo, outfd, pickle.HIGHEST_PROTOCOL)

    return os.path.join(simbasedir,'fakelc-periodsearch.pkl')

This runs periodfinding using several period - finders on a collection of fake LCs .


def periodrec_worker(task):
    '''This is a parallel worker for running period-recovery.

    Parameters
    ----------

    task : tuple
        This is used to pass args to the `periodicvar_recovery` function::

            task[0] = period-finding result pickle to work on
            task[1] = simbasedir
            task[2] = period_tolerance

    Returns
    -------

    dict
        This is the dict produced by the `periodicvar_recovery` function for the
        input period-finding result pickle.

    '''

    pfpkl, simbasedir, period_tolerance = task

    try:
        return periodicvar_recovery(pfpkl,
                                    simbasedir,
                                    period_tolerance=period_tolerance)

    except Exception as e:
        LOGEXCEPTION('periodic var recovery failed for %s' % repr(task))
        return None

def
periodrec_worker
(
task
)
:
pfpkl
,
simbasedir
,
period_tolerance
=
task
try
:
return
periodicvar_recovery
(
pfpkl
,
simbasedir
,
period_tolerance
=
period_tolerance
)
except
Exception
as
e
:
LOGEXCEPTION
(
'periodic var recovery failed for %s'
%
repr
(
task
)
)
return

This is a parallel worker for running period - recovery .


def parallel_periodicvar_recovery(simbasedir,
                                  period_tolerance=1.0e-3,
                                  liststartind=None,
                                  listmaxobjects=None,
                                  nworkers=None):
    '''This is a parallel driver for `periodicvar_recovery`.

    Parameters
    ----------

    simbasedir : str
        The base directory where all of the fake LCs and period-finding results
        are.

    period_tolerance : float
        The maximum difference that this function will consider between an
        actual period (or its aliases) and a recovered period to consider it as
        as a 'recovered' period.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the period-finding result pickles in
        `simbasedir/periodfinding`.

    listmaxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles over several sessions or machines.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    Returns
    -------

    str
        Returns the filename of the pickle produced containing all of the period
        recovery results.

    '''

    # figure out the periodfinding pickles directory
    pfpkldir = os.path.join(simbasedir,'periodfinding')

    if not os.path.exists(pfpkldir):
        LOGERROR('no "periodfinding" subdirectory in %s, can\'t continue' %
                 simbasedir)
        return None

    # find all the periodfinding pickles
    pfpkl_list = glob.glob(os.path.join(pfpkldir,'*periodfinding*pkl*'))

    if len(pfpkl_list) > 0:

        if liststartind:
            pfpkl_list = pfpkl_list[liststartind:]

        if listmaxobjects:
            pfpkl_list = pfpkl_list[:listmaxobjects]

        tasks = [(x, simbasedir, period_tolerance) for x in pfpkl_list]

        pool = mp.Pool(nworkers)
        results = pool.map(periodrec_worker, tasks)
        pool.close()
        pool.join()

        resdict = {x['objectid']:x for x in results if x is not None}

        actual_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and x['actual_vartype'] in PERIODIC_VARTYPES)],
            dtype=np.unicode_
        )

        recovered_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'actual' in x['best_recovered_status'])],
            dtype=np.unicode_
        )
        alias_twice_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'twice' in x['best_recovered_status'])],
            dtype=np.unicode_
        )
        alias_half_periodicvars = np.array(
            [x['objectid'] for x in results
             if (x is not None and 'half' in x['best_recovered_status'])],
            dtype=np.unicode_
        )

        all_objectids = [x['objectid'] for x in results]

        outdict = {'simbasedir':os.path.abspath(simbasedir),
                   'objectids':all_objectids,
                   'period_tolerance':period_tolerance,
                   'actual_periodicvars':actual_periodicvars,
                   'recovered_periodicvars':recovered_periodicvars,
                   'alias_twice_periodicvars':alias_twice_periodicvars,
                   'alias_half_periodicvars':alias_half_periodicvars,
                   'details':resdict}

        outfile = os.path.join(simbasedir,'periodicvar-recovery.pkl')
        with open(outfile, 'wb') as outfd:
            pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)

        return outdict

    else:

        LOGERROR(
            'no periodfinding result pickles found in %s, can\'t continue' %
            pfpkldir
        )
        return None

This is a parallel driver for periodicvar_recovery .


def tic_conesearch(
        ra,
        decl,
        radius_arcmin=5.0,
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=10.0,
        refresh=5.0,
        maxtimeout=90.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''This runs a TESS Input Catalog cone search on MAST.

    If you use this, please cite the TIC paper (Stassun et al 2018;
    http://adsabs.harvard.edu/abs/2018AJ....156..102S). Also see the "living"
    TESS input catalog docs:

    https://docs.google.com/document/d/1zdiKMs4Ld4cXZ2DW4lMX-fuxAF6hPHTjqjIwGqnfjqI

    Also see: https://mast.stsci.edu/api/v0/_t_i_cfields.html for the fields
    returned by the service and present in the result JSON file.

    Parameters
    ----------

    ra,decl : float
        The center coordinates of the cone-search in decimal degrees.

    radius_arcmin : float
        The cone-search radius in arcminutes.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    params = {'ra':ra,
              'dec':decl,
              'radius':radius_arcmin/60.0}
    service = 'Mast.Catalogs.Tic.Cone'

    return mast_query(service,
                      params,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_conesearch ( ra , decl , radius_arcmin = 5.0 , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 10.0 , refresh = 5.0 , maxtimeout = 90.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : params = { 'ra' : ra , 'dec' : decl , 'radius' : radius_arcmin / 60.0 } service = 'Mast.Catalogs.Tic.Cone' return mast_query ( service , params , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )

This runs a TESS Input Catalog cone search on MAST .


def tic_xmatch(
        ra,
        decl,
        radius_arcsec=5.0,
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=90.0,
        refresh=5.0,
        maxtimeout=180.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''This does a cross-match with TIC.

    Parameters
    ----------

    ra,decl : np.arrays or lists of floats
        The coordinates that will be cross-matched against the TIC.

    radius_arcsec : float
        The cross-match radius in arcseconds.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    service = 'Mast.Tic.Crossmatch'

    xmatch_input = {'fields':[{'name':'ra','type':'float'},
                              {'name':'dec','type':'float'}]}
    xmatch_input['data'] = [{'ra':x, 'dec':y} for (x,y) in zip(ra, decl)]

    params = {'raColumn':'ra',
              'decColumn':'dec',
              'radius':radius_arcsec/3600.0}

    return mast_query(service,
                      params,
                      data=xmatch_input,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_xmatch ( ra , decl , radius_arcsec = 5.0 , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 90.0 , refresh = 5.0 , maxtimeout = 180.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : service = 'Mast.Tic.Crossmatch' xmatch_input = { 'fields' : [ { 'name' : 'ra' , 'type' : 'float' } , { 'name' : 'dec' , 'type' : 'float' } ] } xmatch_input [ 'data' ] = [ { 'ra' : x , 'dec' : y } for ( x , y ) in zip ( ra , decl ) ] params = { 'raColumn' : 'ra' , 'decColumn' : 'dec' , 'radius' : radius_arcsec / 3600.0 } return mast_query ( service , params , data = xmatch_input , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )


def tic_objectsearch(
        objectid,
        idcol_to_use="ID",
        apiversion='v0',
        forcefetch=False,
        cachedir='~/.astrobase/mast-cache',
        verbose=True,
        timeout=90.0,
        refresh=5.0,
        maxtimeout=180.0,
        maxtries=3,
        jitter=5.0,
        raiseonfail=False
):
    '''
    This runs a TIC search for a specified TIC ID.

    Parameters
    ----------

    objectid : str
        The object ID to look up information for.

    idcol_to_use : str
        This is the name of the object ID column to use when looking up the
        provided `objectid`. This is one of {'ID', 'HIP', 'TYC', 'UCAC',
        'TWOMASS', 'ALLWISE', 'SDSS', 'GAIA', 'APASS', 'KIC'}.

    apiversion : str
        The API version of the MAST service to use. This sets the URL that this
        function will call, using `apiversion` as key into the `MAST_URLS` dict
        above.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    raiseonfail : bool
        If this is True, the function will raise an Exception if something goes
        wrong, instead of returning None.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}
    '''

    params = {
        'columns':'*',
        'filters':[
            {"paramName": idcol_to_use,
             "values":[str(objectid)]}
        ]
    }
    service = 'Mast.Catalogs.Filtered.Tic'

    return mast_query(service,
                      params,
                      jitter=jitter,
                      apiversion=apiversion,
                      forcefetch=forcefetch,
                      cachedir=cachedir,
                      verbose=verbose,
                      timeout=timeout,
                      refresh=refresh,
                      maxtimeout=maxtimeout,
                      maxtries=maxtries,
                      raiseonfail=raiseonfail)

def tic_objectsearch ( objectid , idcol_to_use = "ID" , apiversion = 'v0' , forcefetch = False , cachedir = '~/.astrobase/mast-cache' , verbose = True , timeout = 90.0 , refresh = 5.0 , maxtimeout = 180.0 , maxtries = 3 , jitter = 5.0 , raiseonfail = False ) : params = { 'columns' : '*' , 'filters' : [ { "paramName" : idcol_to_use , "values" : [ str ( objectid ) ] } ] } service = 'Mast.Catalogs.Filtered.Tic' return mast_query ( service , params , jitter = jitter , apiversion = apiversion , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , raiseonfail = raiseonfail )

This runs a TIC search for a specified TIC ID .


def send_email(sender,
               subject,
               content,
               email_recipient_list,
               email_address_list,
               email_user=None,
               email_pass=None,
               email_server=None):
    '''This sends an email to addresses, informing them about events.

    The email account settings are retrieved from the settings file as described
    above.

    Parameters
    ----------

    sender : str
        The name of the sender to use in the email header.

    subject : str
        Subject of the email.

    content : str
        Content of the email.

    email_recipient list : list of str
        This is a list of email recipient names of the form:
        `['Example Person 1', 'Example Person 1', ...]`

    email_recipient list : list of str
        This is a list of email recipient addresses of the form:
        `['example1@example.com', 'example2@example.org', ...]`

    email_user : str
        The username of the email server account that will send the emails. If
        this is None, the value of EMAIL_USER from the
        ~/.astrobase/.emailsettings file will be used. If that is None as well,
        this function won't work.

    email_pass : str
        The password of the email server account that will send the emails. If
        this is None, the value of EMAIL_PASS from the
        ~/.astrobase/.emailsettings file will be used. If that is None as well,
        this function won't work.

    email_server : str
        The address of the email server that will send the emails. If this is
        None, the value of EMAIL_USER from the ~/.astrobase/.emailsettings file
        will be used. If that is None as well, this function won't work.

    Returns
    -------

    bool
        True if email sending succeeded. False if email sending failed.

    '''

    if not email_user:
        email_user = EMAIL_USER

    if not email_pass:
        email_pass = EMAIL_PASSWORD

    if not email_server:
        email_server = EMAIL_SERVER

    if not email_server and email_user and email_pass:
        raise ValueError("no email server address and "
                         "credentials available, can't continue")

    msg_text = EMAIL_TEMPLATE.format(
        sender=sender,
        hostname=socket.gethostname(),
        activity_time='%sZ' % datetime.utcnow().isoformat(),
        activity_report=content
    )

    email_sender = '%s <%s>' % (sender, EMAIL_USER)

    # put together the recipient and email lists
    email_recipients = [('%s <%s>' % (x,y))
                        for (x,y) in zip(email_recipient_list,
                                         email_address_list)]

    # put together the rest of the message
    email_msg = MIMEText(msg_text)
    email_msg['From'] = email_sender
    email_msg['To'] = ', '.join(email_recipients)
    email_msg['Message-Id'] = make_msgid()
    email_msg['Subject'] = '[%s on %s] %s' % (
        sender,
        socket.gethostname(),
        subject
    )
    email_msg['Date'] = formatdate(time.time())

    # start the email process

    try:
        server = smtplib.SMTP(EMAIL_SERVER, 587)
        server_ehlo_response = server.ehlo()

        if server.has_extn('STARTTLS'):

            try:

                tls_start_response = server.starttls()
                tls_ehlo_response = server.ehlo()

                login_response = server.login(EMAIL_USER, EMAIL_PASSWORD)

                send_response = (
                    server.sendmail(email_sender,
                                    email_address_list,
                                    email_msg.as_string())
                )

            except Exception as e:

                print('script email sending failed with error: %s'
                      % e)
                send_response = None

            if send_response is not None:

                print('script email sent successfully')
                quit_response = server.quit()
                return True

            else:

                quit_response = server.quit()
                return False

        else:

            print('email server does not support STARTTLS,'
                  ' bailing out...')
            quit_response = server.quit()
            return False

    except Exception as e:
        print('sending email failed with error: %s' % e)
        returnval = False

    quit_response = server.quit()
    return returnval

This sends an email to addresses informing them about events .


def fourier_sinusoidal_func(fourierparams, times, mags, errs):
    '''This generates a sinusoidal light curve using a Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate model
        mags, and the input `times`, `mags`, and `errs` will be resorted by
        model phase and returned.

    Returns
    -------

    (modelmags, phase, ptimes, pmags, perrs) : tuple
        Returns the model mags and phase values. Also returns the input `times`,
        `mags`, and `errs` sorted by the model's phase.

    '''

    period, epoch, famps, fphases = fourierparams

    # figure out the order from the length of the Fourier param list
    forder = len(famps)

    # phase the times with this period
    iphase = (times - epoch)/period
    iphase = iphase - np.floor(iphase)

    phasesortind = np.argsort(iphase)
    phase = iphase[phasesortind]
    ptimes = times[phasesortind]
    pmags = mags[phasesortind]
    perrs = errs[phasesortind]

    # calculate all the individual terms of the series
    fseries = [famps[x]*np.cos(2.0*np.pi*x*phase + fphases[x])
               for x in range(forder)]

    # this is the zeroth order coefficient - a constant equal to median mag
    modelmags = np.median(mags)

    # sum the series
    for fo in fseries:
        modelmags += fo

    return modelmags, phase, ptimes, pmags, perrs

This generates a sinusoidal light curve using a Fourier cosine series .


def fourier_sinusoidal_residual(fourierparams, times, mags, errs):
    '''
    This returns the residual between the model mags and the actual mags.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    times,mags,errs : np.array
        The input time-series of measurements and associated errors for which
        the model will be generated. The times will be used to generate model
        mags, and the input `times`, `mags`, and `errs` will be resorted by
        model phase and returned.

    Returns
    -------

    np.array
        The residuals between the input `mags` and generated `modelmags`,
        weighted by the measurement errors in `errs`.

    '''
    modelmags, phase, ptimes, pmags, perrs = (
        fourier_sinusoidal_func(fourierparams, times, mags, errs)
    )

    # this is now a weighted residual taking into account the measurement err
    return (pmags - modelmags)/perrs

This returns the residual between the model mags and the actual mags .