def pklc_fovcatalog_objectinfo(
        pklcdir,
        fovcatalog,
        fovcatalog_columns=[0,1,2,
                            6,7,
                            8,9,
                            10,11,
                            13,14,15,16,
                            17,18,19,
                            20,21],
        fovcatalog_colnames=['objectid','ra','decl',
                             'jmag','jmag_err',
                             'hmag','hmag_err',
                             'kmag','kmag_err',
                             'bmag','vmag','rmag','imag',
                             'sdssu','sdssg','sdssr',
                             'sdssi','sdssz'],
        fovcatalog_colformats=('U20,f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,f8,f8,'
                               'f8,f8,f8,'
                               'f8,f8')
):
    '''Adds catalog info to objectinfo key of all pklcs in lcdir.

    If fovcatalog, fovcatalog_columns, fovcatalog_colnames are provided, uses
    them to find all the additional information listed in the fovcatalog_colname
    keys, and writes this info to the objectinfo key of each lcdict. This makes
    it easier for astrobase tools to work on these light curve.

    The default set up for fovcatalog is to use a text file generated by the
    HATPI pipeline before auto-calibrating a field. The format is specified as
    above in _columns,  _colnames, and _colformats.

    '''

    if fovcatalog.endswith('.gz'):
        catfd = gzip.open(fovcatalog)
    else:
        catfd = open(fovcatalog)

    # read the catalog using the colformats, etc.
    fovcat = np.genfromtxt(catfd,
                           usecols=fovcatalog_columns,
                           names=fovcatalog_colnames,
                           dtype=fovcatalog_colformats)
    catfd.close()

    pklclist = sorted(glob.glob(os.path.join(pklcdir, '*HAT*-pklc.pkl')))

    updatedpklcs, failedpklcs = [], []

    for pklc in pklclist:

        lcdict = read_hatpi_pklc(pklc)
        objectid = lcdict['objectid']

        catind = np.where(fovcat['objectid'] == objectid)

        # if we found catalog info for this object, put it into objectinfo
        if len(catind) > 0 and catind[0]:

            lcdict['objectinfo'].update(
                {x:y for x,y in zip(
                    fovcatalog_colnames,
                    [np.asscalar(fovcat[z][catind]) for
                     z in fovcatalog_colnames]
                )
                }
            )

            # write the LC back to the pickle (tempfile for atomicity)
            with open(pklc+'-tmp','wb') as outfd:
                pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)

            # overwrite previous once we know it exists
            if os.path.exists(pklc+'-tmp'):
                shutil.move(pklc+'-tmp',pklc)

                LOGINFO('updated %s with catalog info for %s at %.3f, %.3f OK' %
                        (pklc, objectid,
                         lcdict['objectinfo']['ra'],
                         lcdict['objectinfo']['decl']))

                updatedpklcs.append(pklc)

        # otherwise, do nothing
        else:
            failedpklcs.append(pklc)

    # end of pklclist processing
    return updatedpklcs, failedpklcs

Adds catalog info to objectinfo key of all pklcs in lcdir .


def _base64_to_file(b64str, outfpath, writetostrio=False):
    '''This converts the base64 encoded string to a file.

    Parameters
    ----------

    b64str : str
        A base64 encoded strin that is the output of `base64.b64encode`.

    outfpath : str
        The path to where the file will be written. This should include an
        appropriate extension for the file (e.g. a base64 encoded string that
        represents a PNG should have its `outfpath` end in a '.png') so the OS
        can open these files correctly.

    writetostrio : bool
        If this is True, will return a StringIO object with the binary stream
        decoded from the base64-encoded input string `b64str`. This can be
        useful to embed these into other files without having to write them to
        disk.

    Returns
    -------

    str or StringIO object
        If `writetostrio` is False, will return the output file's path as a
        str. If it is True, will return a StringIO object directly. If writing
        the file fails in either case, will return None.

    '''

    try:

        filebytes = base64.b64decode(b64str)

        # if we're writing back to a stringio object
        if writetostrio:

            outobj = StrIO(filebytes)
            return outobj

        # otherwise, we're writing to an actual file
        else:

            with open(outfpath,'wb') as outfd:
                outfd.write(filebytes)

            if os.path.exists(outfpath):
                return outfpath
            else:
                LOGERROR('could not write output file: %s' % outfpath)
                return None

    except Exception as e:

        LOGEXCEPTION('failed while trying to convert '
                     'b64 string to file %s' % outfpath)
        return None

This converts the base64 encoded string to a file .


def _read_checkplot_picklefile(checkplotpickle):
    '''This reads a checkplot gzipped pickle file back into a dict.

    NOTE: the try-except is for Python 2 pickles that have numpy arrays in
    them. Apparently, these aren't compatible with Python 3. See here:

    http://stackoverflow.com/q/11305790

    The workaround is noted in this answer:

    http://stackoverflow.com/a/41366785

    Parameters
    ----------

    checkplotpickle : str
        The path to a checkplot pickle file. This can be a gzipped file (in
        which case the file extension should end in '.gz')

    Returns
    -------

    dict
        This returns a checkplotdict.

    '''

    if checkplotpickle.endswith('.gz'):

        try:
            with gzip.open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd)

        except UnicodeDecodeError:

            with gzip.open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd, encoding='latin1')

    else:

        try:
            with open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd)

        except UnicodeDecodeError:

            with open(checkplotpickle,'rb') as infd:
                cpdict = pickle.load(infd, encoding='latin1')

    return cpdict

def _read_checkplot_picklefile ( checkplotpickle ) : if checkplotpickle . endswith ( '.gz' ) : try : with gzip . open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd ) except UnicodeDecodeError : with gzip . open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd , encoding = 'latin1' ) else : try : with open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd ) except UnicodeDecodeError : with open ( checkplotpickle , 'rb' ) as infd : cpdict = pickle . load ( infd , encoding = 'latin1' ) return cpdict

This reads a checkplot gzipped pickle file back into a dict .


def make_fit_plot(phase, pmags, perrs, fitmags,
                  period, mintime, magseriesepoch,
                  plotfit,
                  magsarefluxes=False,
                  wrap=False,
                  model_over_lc=False):
    '''This makes a plot of the LC model fit.

    Parameters
    ----------

    phase,pmags,perrs : np.array
        The actual mag/flux time-series.

    fitmags : np.array
        The model fit time-series.

    period : float
        The period at which the phased LC was generated.

    mintime : float
        The minimum time value.

    magseriesepoch : float
        The value of time around which the phased LC was folded.

    plotfit : str
        The name of a file to write the plot to.

    magsarefluxes : bool
        Set this to True if the values in `pmags` and `fitmags` are actually
        fluxes.

    wrap : bool
        If True, will wrap the phased LC around 0.0 to make some phased LCs
        easier to look at.

    model_over_lc : bool
        Usually, this function will plot the actual LC over the model LC. Set
        this to True to plot the model over the actual LC; this is most useful
        when you have a very dense light curve and want to be able to see how it
        follows the model.

    Returns
    -------

    Nothing.

    '''

    # set up the figure
    plt.close('all')
    plt.figure(figsize=(8,4.8))

    if model_over_lc:
        model_z = 100
        lc_z = 0
    else:
        model_z = 0
        lc_z = 100

    if not wrap:

        plt.plot(phase, fitmags, linewidth=3.0, color='red',zorder=model_z)
        plt.plot(phase,pmags,
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=lc_z)

        # set the x axis ticks and label
        plt.gca().set_xticks(
            [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]
        )

    else:
        plt.plot(np.concatenate([phase-1.0,phase]),
                 np.concatenate([fitmags,fitmags]),
                 linewidth=3.0,
                 color='red',zorder=model_z)
        plt.plot(np.concatenate([phase-1.0,phase]),
                 np.concatenate([pmags,pmags]),
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=lc_z)

        plt.gca().set_xlim((-0.8,0.8))
        # set the x axis ticks and label
        plt.gca().set_xticks(
            [-0.8,-0.7,-0.6,-0.5,-0.4,-0.3,-0.2,-0.1,
             0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]
        )

    # set the y axis limit and label
    ymin, ymax = plt.ylim()
    if not magsarefluxes:
        plt.gca().invert_yaxis()
        plt.ylabel('magnitude')
    else:
        plt.ylabel('flux')

    plt.xlabel('phase')
    plt.title('period: %.6f, folded at %.6f, fit epoch: %.6f' %
              (period, mintime, magseriesepoch))
    plt.savefig(plotfit)
    plt.close()


def objectlist_conesearch(racenter,
                          declcenter,
                          searchradiusarcsec,
                          gaia_mirror=None,
                          columns=('source_id',
                                   'ra','dec',
                                   'phot_g_mean_mag',
                                   'l','b',
                                   'parallax', 'parallax_error',
                                   'pmra','pmra_error',
                                   'pmdec','pmdec_error'),
                          extra_filter=None,
                          returnformat='csv',
                          forcefetch=False,
                          cachedir='~/.astrobase/gaia-cache',
                          verbose=True,
                          timeout=15.0,
                          refresh=2.0,
                          maxtimeout=300.0,
                          maxtries=3,
                          complete_query_later=True):
    '''This queries the GAIA TAP service for a list of objects near the coords.

    Runs a conesearch around `(racenter, declcenter)` with radius in arcsec of
    `searchradiusarcsec`.

    Parameters
    ----------

    racenter,declcenter : float
        The center equatorial coordinates in decimal degrees.

    searchradiusarcsec : float
        The search radius of the cone-search in arcseconds.

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    extra_filter: str or None
        If this is provided, must be a valid ADQL filter string that is used to
        further filter the cone-search results.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # this was generated using the awesome query generator at:
    # https://gea.esac.esa.int/archive/

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns}, "
        "(DISTANCE(POINT('ICRS', "
        "{{table}}.ra, {{table}}.dec), "
        "POINT('ICRS', {ra_center:.5f}, {decl_center:.5f})))*3600.0 "
        "AS dist_arcsec "
        "from {{table}} where "
        "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec),"
        "CIRCLE('ICRS',{ra_center:.5f},{decl_center:.5f},"
        "{search_radius:.6f}))=1 "
        "{extra_filter_str}"
        "ORDER by dist_arcsec asc "
    )

    if extra_filter is not None:
        extra_filter_str = ' and %s ' % extra_filter
    else:
        extra_filter_str = ''

    formatted_query = query.format(ra_center=racenter,
                                   decl_center=declcenter,
                                   search_radius=searchradiusarcsec/3600.0,
                                   extra_filter_str=extra_filter_str,
                                   columns=', '.join(columns))

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a list of objects near the coords .


def objectlist_radeclbox(radeclbox,
                         gaia_mirror=None,
                         columns=('source_id',
                                  'ra','dec',
                                  'phot_g_mean_mag',
                                  'l','b',
                                  'parallax, parallax_error',
                                  'pmra','pmra_error',
                                  'pmdec','pmdec_error'),
                         extra_filter=None,
                         returnformat='csv',
                         forcefetch=False,
                         cachedir='~/.astrobase/gaia-cache',
                         verbose=True,
                         timeout=15.0,
                         refresh=2.0,
                         maxtimeout=300.0,
                         maxtries=3,
                         complete_query_later=True):

    '''This queries the GAIA TAP service for a list of objects in an equatorial
    coordinate box.

    Parameters
    ----------

    radeclbox : sequence of four floats
        This defines the box to search in::

            [ra_min, ra_max, decl_min, decl_max]

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    extra_filter: str or None
        If this is provided, must be a valid ADQL filter string that is used to
        further filter the cone-search results.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # this was generated using the awesome query generator at:
    # https://gea.esac.esa.int/archive/

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns} from {{table}} where "
        "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec),"
        "BOX('ICRS',{ra_center:.5f},{decl_center:.5f},"
        "{ra_width:.5f},{decl_height:.5f}))=1"
        "{extra_filter_str}"
    )

    ra_min, ra_max, decl_min, decl_max = radeclbox
    ra_center = (ra_max + ra_min)/2.0
    decl_center = (decl_max + decl_min)/2.0
    ra_width = ra_max - ra_min
    decl_height = decl_max - decl_min

    if extra_filter is not None:
        extra_filter_str = ' and %s ' % extra_filter
    else:
        extra_filter_str = ''

    formatted_query = query.format(columns=', '.join(columns),
                                   extra_filter_str=extra_filter_str,
                                   ra_center=ra_center,
                                   decl_center=decl_center,
                                   ra_width=ra_width,
                                   decl_height=decl_height)

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a list of objects in an equatorial coordinate box .


def objectid_search(gaiaid,
                    gaia_mirror=None,
                    columns=('source_id',
                             'ra','dec',
                             'phot_g_mean_mag',
                             'phot_bp_mean_mag',
                             'phot_rp_mean_mag',
                             'l','b',
                             'parallax, parallax_error',
                             'pmra','pmra_error',
                             'pmdec','pmdec_error'),
                    returnformat='csv',
                    forcefetch=False,
                    cachedir='~/.astrobase/gaia-cache',
                    verbose=True,
                    timeout=15.0,
                    refresh=2.0,
                    maxtimeout=300.0,
                    maxtries=3,
                    complete_query_later=True):

    '''This queries the GAIA TAP service for a single GAIA source ID.

    Parameters
    ----------

    gaiaid : str
        The source ID of the object whose info will be collected.

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    query = (
        "select {columns} from {{table}} where "
        "source_id = {gaiaid}"
    )

    formatted_query = query.format(columns=', '.join(columns),
                                   gaiaid=gaiaid)

    return tap_query(formatted_query,
                     gaia_mirror=gaia_mirror,
                     returnformat=returnformat,
                     forcefetch=forcefetch,
                     cachedir=cachedir,
                     verbose=verbose,
                     timeout=timeout,
                     refresh=refresh,
                     maxtimeout=maxtimeout,
                     maxtries=maxtries,
                     complete_query_later=complete_query_later)

This queries the GAIA TAP service for a single GAIA source ID .


def generalized_lsp_value_notau(times, mags, errs, omega):
    '''
    This is the simplified version not using tau.

    The relations used are::

        W = sum (1.0/(errs*errs) )
        w_i = (1/W)*(1/(errs*errs))

        Y = sum( w_i*y_i )
        C = sum( w_i*cos(wt_i) )
        S = sum( w_i*sin(wt_i) )

        YY = sum( w_i*y_i*y_i ) - Y*Y
        YC = sum( w_i*y_i*cos(wt_i) ) - Y*C
        YS = sum( w_i*y_i*sin(wt_i) ) - Y*S

        CpC = sum( w_i*cos(w_t_i)*cos(w_t_i) )
        CC = CpC - C*C
        SS = (1 - CpC) - S*S
        CS = sum( w_i*cos(w_t_i)*sin(w_t_i) ) - C*S

        D(omega) = CC*SS - CS*CS
        P(omega) = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*D)

    Parameters
    ----------

    times,mags,errs : np.array
        The time-series to calculate the periodogram value for.

    omega : float
        The frequency to calculate the periodogram value at.

    Returns
    -------

    periodogramvalue : float
        The normalized periodogram at the specified test frequency `omega`.

    '''

    one_over_errs2 = 1.0/(errs*errs)

    W = npsum(one_over_errs2)
    wi = one_over_errs2/W

    sin_omegat = npsin(omega*times)
    cos_omegat = npcos(omega*times)

    sin2_omegat = sin_omegat*sin_omegat
    cos2_omegat = cos_omegat*cos_omegat
    sincos_omegat = sin_omegat*cos_omegat

    # calculate some more sums and terms
    Y = npsum( wi*mags )
    C = npsum( wi*cos_omegat )
    S = npsum( wi*sin_omegat )

    YpY = npsum( wi*mags*mags)

    YpC = npsum( wi*mags*cos_omegat )
    YpS = npsum( wi*mags*sin_omegat )

    CpC = npsum( wi*cos2_omegat )
    # SpS = npsum( wi*sin2_omegat )

    CpS = npsum( wi*sincos_omegat )

    # the final terms
    YY = YpY - Y*Y
    YC = YpC - Y*C
    YS = YpS - Y*S
    CC = CpC - C*C
    SS = 1 - CpC - S*S  # use SpS = 1 - CpC
    CS = CpS - C*S

    # P(omega) = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*D)
    # D(omega) = CC*SS - CS*CS
    Domega = CC*SS - CS*CS
    lspval = (SS*YC*YC + CC*YS*YS - 2.0*CS*YC*YS)/(YY*Domega)

    return lspval

This is the simplified version not using tau .


def specwindow_lsp_value(times, mags, errs, omega):
    '''This calculates the peak associated with the spectral window function
    for times and at the specified omega.

    NOTE: this is classical Lomb-Scargle, not the Generalized
    Lomb-Scargle. `mags` and `errs` are silently ignored since we're calculating
    the periodogram of the observing window function. These are kept to present
    a consistent external API so the `pgen_lsp` function below can call this
    transparently.

    Parameters
    ----------

    times,mags,errs : np.array
        The time-series to calculate the periodogram value for.

    omega : float
        The frequency to calculate the periodogram value at.

    Returns
    -------

    periodogramvalue : float
        The normalized periodogram at the specified test frequency `omega`.

    '''

    norm_times = times - times.min()

    tau = (
        (1.0/(2.0*omega)) *
        nparctan( npsum(npsin(2.0*omega*norm_times)) /
                  npsum(npcos(2.0*omega*norm_times)) )
    )

    lspval_top_cos = (npsum(1.0 * npcos(omega*(norm_times-tau))) *
                      npsum(1.0 * npcos(omega*(norm_times-tau))))
    lspval_bot_cos = npsum( (npcos(omega*(norm_times-tau))) *
                            (npcos(omega*(norm_times-tau))) )

    lspval_top_sin = (npsum(1.0 * npsin(omega*(norm_times-tau))) *
                      npsum(1.0 * npsin(omega*(norm_times-tau))))
    lspval_bot_sin = npsum( (npsin(omega*(norm_times-tau))) *
                            (npsin(omega*(norm_times-tau))) )

    lspval = 0.5 * ( (lspval_top_cos/lspval_bot_cos) +
                     (lspval_top_sin/lspval_bot_sin) )

    return lspval

def specwindow_lsp_value ( times , mags , errs , omega ) : norm_times = times - times . min ( ) tau = ( ( 1.0 / ( 2.0 * omega ) ) * nparctan ( npsum ( npsin ( 2.0 * omega * norm_times ) ) / npsum ( npcos ( 2.0 * omega * norm_times ) ) ) ) lspval_top_cos = ( npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) ) lspval_bot_cos = npsum ( ( npcos ( omega * ( norm_times - tau ) ) ) * ( npcos ( omega * ( norm_times - tau ) ) ) ) lspval_top_sin = ( npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) ) lspval_bot_sin = npsum ( ( npsin ( omega * ( norm_times - tau ) ) ) * ( npsin ( omega * ( norm_times - tau ) ) ) ) lspval = 0.5 * ( ( lspval_top_cos / lspval_bot_cos ) + ( lspval_top_sin / lspval_bot_sin ) ) return lspval

This calculates the peak associated with the spectral window function for times and at the specified omega .


def specwindow_lsp(
        times,
        mags,
        errs,
        magsarefluxes=False,
        startp=None,
        endp=None,
        stepsize=1.0e-4,
        autofreq=True,
        nbestpeaks=5,
        periodepsilon=0.1,
        sigclip=10.0,
        nworkers=None,
        glspfunc=_glsp_worker_specwindow,
        verbose=True
):
    '''This calculates the spectral window function.

    Wraps the `pgen_lsp` function above to use the specific worker for
    calculating the window-function.

    Parameters
    ----------

    times,mags,errs : np.array
        The mag/flux time-series with associated measurement errors to run the
        period-finding on.

    magsarefluxes : bool
        If the input measurement values in `mags` and `errs` are in fluxes, set
        this to True.

    startp,endp : float or None
        The minimum and maximum periods to consider for the transit search.

    stepsize : float
        The step-size in frequency to use when constructing a frequency grid for
        the period search.

    autofreq : bool
        If this is True, the value of `stepsize` will be ignored and the
        :py:func:`astrobase.periodbase.get_frequency_grid` function will be used
        to generate a frequency grid based on `startp`, and `endp`. If these are
        None as well, `startp` will be set to 0.1 and `endp` will be set to
        `times.max() - times.min()`.

    nbestpeaks : int
        The number of 'best' peaks to return from the periodogram results,
        starting from the global maximum of the periodogram peak values.

    periodepsilon : float
        The fractional difference between successive values of 'best' periods
        when sorting by periodogram power to consider them as separate periods
        (as opposed to part of the same periodogram peak). This is used to avoid
        broad peaks in the periodogram and make sure the 'best' periods returned
        are all actually independent.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nworkers : int
        The number of parallel workers to use when calculating the periodogram.

    glspfunc : Python function
        The worker function to use to calculate the periodogram. This is used to
        used to make the `pgen_lsp` function calculate the time-series sampling
        window function instead of the time-series measurements' GLS periodogram
        by passing in `_glsp_worker_specwindow` instead of the default
        `_glsp_worker` function.

    verbose : bool
        If this is True, will indicate progress and details about the frequency
        grid used for the period search.

    Returns
    -------

    dict
        This function returns a dict, referred to as an `lspinfo` dict in other
        astrobase functions that operate on periodogram results. This is a
        standardized format across all astrobase period-finders, and is of the
        form below::

            {'bestperiod': the best period value in the periodogram,
             'bestlspval': the periodogram peak associated with the best period,
             'nbestpeaks': the input value of nbestpeaks,
             'nbestlspvals': nbestpeaks-size list of best period peak values,
             'nbestperiods': nbestpeaks-size list of best periods,
             'lspvals': the full array of periodogram powers,
             'periods': the full array of periods considered,
             'method':'win' -> the name of the period-finder method,
             'kwargs':{ dict of all of the input kwargs for record-keeping}}

    '''

    # run the LSP using glsp_worker_specwindow as the worker
    lspres = pgen_lsp(
        times,
        mags,
        errs,
        magsarefluxes=magsarefluxes,
        startp=startp,
        endp=endp,
        autofreq=autofreq,
        nbestpeaks=nbestpeaks,
        periodepsilon=periodepsilon,
        stepsize=stepsize,
        nworkers=nworkers,
        sigclip=sigclip,
        glspfunc=glspfunc,
        verbose=verbose
    )

    # update the resultdict to indicate we're a spectral window function
    lspres['method'] = 'win'

    if lspres['lspvals'] is not None:

        # renormalize the periodogram to between 0 and 1 like the usual GLS.
        lspmax = npnanmax(lspres['lspvals'])

        if npisfinite(lspmax):

            lspres['lspvals'] = lspres['lspvals']/lspmax
            lspres['nbestlspvals'] = [
                x/lspmax for x in lspres['nbestlspvals']
            ]
            lspres['bestlspval'] = lspres['bestlspval']/lspmax

    return lspres

This calculates the spectral window function .


def check_existing_apikey(lcc_server):
    '''This validates if an API key for the specified LCC-Server is available.

    API keys are stored using the following file scheme::

        ~/.astrobase/lccs/apikey-domain.of.lccserver.org

    e.g. for the HAT LCC-Server at https://data.hatsurveys.org::

        ~/.astrobase/lccs/apikey-https-data.hatsurveys.org

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server for which the existence of API keys will
        be checked.

    Returns
    -------

    (apikey_ok, apikey_str, expiry) : tuple
        The returned tuple contains the status of the API key, the API key
        itself if present, and its expiry date if present.

    '''

    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    if os.path.exists(APIKEYFILE):

        # check if this file is readable/writeable by user only
        fileperm = oct(os.stat(APIKEYFILE)[stat.ST_MODE])

        if fileperm == '0100600' or fileperm == '0o100600':

            with open(APIKEYFILE) as infd:
                apikey, expires = infd.read().strip('\n').split()

            # get today's datetime
            now = datetime.now(utc)

            if sys.version_info[:2] < (3,7):
                # this hideous incantation is required for lesser Pythons
                expdt = datetime.strptime(
                    expires.replace('Z',''),
                    '%Y-%m-%dT%H:%M:%S.%f'
                ).replace(tzinfo=utc)
            else:
                expdt = datetime.fromisoformat(expires.replace('Z','+00:00'))

            if now > expdt:
                LOGERROR('API key has expired. expiry was on: %s' % expires)
                return False, apikey, expires
            else:
                return True, apikey, expires

        else:
            LOGWARNING('The API key file %s has bad permissions '
                       'and is insecure, not reading it.\n'
                       '(you need to chmod 600 this file)'
                       % APIKEYFILE)

            return False, None, None
    else:
        LOGWARNING('No LCC-Server API key '
                   'found in: {apikeyfile}'.format(apikeyfile=APIKEYFILE))

        return False, None, None

This validates if an API key for the specified LCC - Server is available .


def get_new_apikey(lcc_server):
    '''This gets a new API key from the specified LCC-Server.

    NOTE: this only gets an anonymous API key. To get an API key tied to a user
    account (and associated privilege level), see the `import_apikey` function
    below.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server from where the API key will be fetched.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''

    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    # url for getting an API key
    url = '%s/api/key' % lcc_server

    # get the API key
    resp = urlopen(url)

    if resp.code == 200:

        respdict = json.loads(resp.read())

    else:

        LOGERROR('could not fetch the API key from LCC-Server at: %s' %
                 lcc_server)
        LOGERROR('the HTTP status code was: %s' % resp.status_code)
        return None

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    apikey = respdict['result']['apikey']
    expires = respdict['result']['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(APIKEYFILE)):
        os.makedirs(os.path.dirname(APIKEYFILE))

    with open(APIKEYFILE,'w') as outfd:
        outfd.write('%s %s\n' % (apikey, expires))

    # chmod it to the correct value
    os.chmod(APIKEYFILE, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (lcc_server,
                                                                   expires))
    LOGINFO('written to: %s' % APIKEYFILE)

    return apikey, expires

This gets a new API key from the specified LCC - Server .


def import_apikey(lcc_server, apikey_text_json):
    '''This imports an API key from text and writes it to the cache dir.

    Use this with the JSON text copied from the API key text box on your
    LCC-Server user home page. The API key will thus be tied to the privileges
    of that user account and can then access objects, datasets, and collections
    marked as private for the user only or shared with that user.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server to get the API key for.

    apikey_text_json : str
        The JSON string from the API key text box on the user's LCC-Server home
        page at `lcc_server/users/home`.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''
    USERHOME = os.path.expanduser('~')
    APIKEYFILE = os.path.join(USERHOME,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % lcc_server.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    respdict = json.loads(apikey_text_json)

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    apikey = respdict['apikey']
    expires = respdict['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(APIKEYFILE)):
        os.makedirs(os.path.dirname(APIKEYFILE))

    with open(APIKEYFILE,'w') as outfd:
        outfd.write('%s %s\n' % (apikey, expires))

    # chmod it to the correct value
    os.chmod(APIKEYFILE, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (lcc_server,
                                                                   expires))
    LOGINFO('written to: %s' % APIKEYFILE)

    return apikey, expires

This imports an API key from text and writes it to the cache dir .


def submit_post_searchquery(url, data, apikey):
    '''This submits a POST query to an LCC-Server search API endpoint.

    Handles streaming of the results, and returns the final JSON stream. Also
    handles results that time out.

    Parameters
    ----------

    url : str
        The URL of the search API endpoint to hit. This is something like
        `https://data.hatsurveys.org/api/conesearch`

    data : dict
        A dict of the search query parameters to pass to the search service.

    apikey : str
        The API key to use to access the search service. API keys are required
        for all POST request made to an LCC-Server's API endpoints.

    Returns
    -------

    (status_flag, data_dict, dataset_id) : tuple
        This returns a tuple containing the status of the request: ('complete',
        'failed', 'background', etc.), a dict parsed from the JSON result of the
        request, and a dataset ID, which can be used to reconstruct the URL on
        the LCC-Server where the results can be browsed.

    '''

    # first, we need to convert any columns and collections items to broken out
    # params
    postdata = {}

    for key in data:

        if key == 'columns':
            postdata['columns[]'] = data[key]
        elif key == 'collections':
            postdata['collections[]'] = data[key]
        else:
            postdata[key] = data[key]

    # do the urlencode with doseq=True
    # we also need to encode to bytes
    encoded_postdata = urlencode(postdata, doseq=True).encode()

    # if apikey is not None, add it in as an Authorization: Bearer [apikey]
    # header
    if apikey:
        headers = {'Authorization':'Bearer: %s' % apikey}
    else:
        headers = {}

    LOGINFO('submitting search query to LCC-Server API URL: %s' % url)

    try:

        # hit the server with a POST request
        req = Request(url, data=encoded_postdata, headers=headers)
        resp = urlopen(req)

        if resp.code == 200:

            # we'll iterate over the lines in the response
            # this works super-well for ND-JSON!
            for line in resp:

                data = json.loads(line)
                msg = data['message']
                status = data['status']

                if status != 'failed':
                    LOGINFO('status: %s, %s' % (status, msg))
                else:
                    LOGERROR('status: %s, %s' % (status, msg))

                # here, we'll decide what to do about the query

                # completed query or query sent to background...
                if status in ('ok','background'):

                    setid = data['result']['setid']
                    # save the data pickle to astrobase lccs directory
                    outpickle = os.path.join(os.path.expanduser('~'),
                                             '.astrobase',
                                             'lccs',
                                             'query-%s.pkl' % setid)
                    if not os.path.exists(os.path.dirname(outpickle)):
                        os.makedirs(os.path.dirname(outpickle))

                    with open(outpickle,'wb') as outfd:
                        pickle.dump(data, outfd, pickle.HIGHEST_PROTOCOL)
                        LOGINFO('saved query info to %s, use this to '
                                'download results later with '
                                'retrieve_dataset_files' % outpickle)

                    # we're done at this point, return
                    return status, data, data['result']['setid']

                # the query probably failed...
                elif status == 'failed':

                    # we're done at this point, return
                    return status, data, None

        # if the response was not OK, then we probably failed
        else:

            try:
                data = json.load(resp)
                msg = data['message']

                LOGERROR(msg)
                return 'failed', None, None

            except Exception as e:

                LOGEXCEPTION('failed to submit query to %s' % url)
                return 'failed', None, None

    except HTTPError as e:

        LOGERROR('could not submit query to LCC API at: %s' % url)
        LOGERROR('HTTP status code was %s, reason: %s' % (e.code, e.reason))
        return 'failed', None, None

This submits a POST query to an LCC - Server search API endpoint .


def cone_search(lcc_server,
                center_ra,
                center_decl,
                radiusarcmin=5.0,
                result_visibility='unlisted',
                email_when_done=False,
                collections=None,
                columns=None,
                filters=None,
                sortspec=None,
                samplespec=None,
                limitspec=None,
                download_data=True,
                outdir=None,
                maxtimeout=300.0,
                refresh=15.0):

    '''This runs a cone-search query.

    Parameters
    ----------

    lcc_server : str
        This is the base URL of the LCC-Server to talk to.  (e.g. for HAT, use:
        https://data.hatsurveys.org)

    center_ra,center_decl : float
        These are the central coordinates of the search to conduct. These can be
        either decimal degrees of type float, or sexagesimal coordinates of type
        str:

        - OK: 290.0, 45.0
        - OK: 15:00:00 +45:00:00
        - OK: 15 00 00.0 -45 00 00.0
        - NOT OK: 290.0 +45:00:00
        - NOT OK: 15:00:00 45.0

    radiusarcmin : float
        This is the search radius to use for the cone-search. This is in
        arcminutes. The maximum radius you can use is 60 arcminutes = 1 degree.

    result_visibility : {'private', 'unlisted', 'public'}
        This sets the visibility of the dataset produced from the search
        result::

               'private' -> the dataset and its products are not visible or
                            accessible by any user other than the one that
                            created the dataset.

               'unlisted' -> the dataset and its products are not visible in the
                             list of public datasets, but can be accessed if the
                             dataset URL is known

               'public' -> the dataset and its products are visible in the list
                           of public datasets and can be accessed by anyone.

    email_when_done : bool
        If True, the LCC-Server will email you when the search is complete. This
        will also set `download_data` to False. Using this requires an
        LCC-Server account and an API key tied to that account.

    collections : list of str or None
        This is a list of LC collections to search in. If this is None, all
        collections will be searched.

    columns : list of str or None
        This is a list of columns to return in the results. Matching objects'
        object IDs, RAs, DECs, and links to light curve files will always be
        returned so there is no need to specify these columns. If None, only
        these columns will be returned: 'objectid', 'ra', 'decl', 'lcfname'

    filters : str or None
        This is an SQL-like string to use to filter on database columns in the
        LCC-Server's collections. To see the columns available for a search,
        visit the Collections tab in the LCC-Server's browser UI. The filter
        operators allowed are::

            lt      -> less than
            gt      -> greater than
            ge      -> greater than or equal to
            le      -> less than or equal to
            eq      -> equal to
            ne      -> not equal to
            ct      -> contains text
            isnull  -> column value is null
            notnull -> column value is not null

        You may use the `and` and `or` operators between filter specifications
        to chain them together logically.

        Example filter strings::

            "(propermotion gt 200.0) and (sdssr lt 11.0)"
            "(dered_jmag_kmag gt 2.0) and (aep_000_stetsonj gt 10.0)"
            "(gaia_status ct 'ok') and (propermotion gt 300.0)"
            "(simbad_best_objtype ct 'RR') and (dered_sdssu_sdssg lt 0.5)"

    sortspec : tuple of two strs or None
        If not None, this should be a tuple of two items::

            ('column to sort by', 'asc|desc')

        This sets the column to sort the results by. For cone_search, the
        default column and sort order are 'dist_arcsec' and 'asc', meaning the
        distance from the search center in ascending order.

    samplespec : int or None
        If this is an int, will indicate how many rows from the initial search
        result will be uniformly random sampled and returned.

    limitspec : int or None
        If this is an int, will indicate how many rows from the initial search
        result to return in total.

        `sortspec`, `samplespec`, and `limitspec` are applied in this order:

            sample -> sort -> limit

    download_data : bool
        This sets if the accompanying data from the search results will be
        downloaded automatically. This includes the data table CSV, the dataset
        pickle file, and a light curve ZIP file. Note that if the search service
        indicates that your query is still in progress, this function will block
        until the light curve ZIP file becomes available. The maximum wait time
        in seconds is set by maxtimeout and the refresh interval is set by
        refresh.

        To avoid the wait block, set download_data to False and the function
        will write a pickle file to `~/.astrobase/lccs/query-[setid].pkl`
        containing all the information necessary to retrieve these data files
        later when the query is done. To do so, call the
        `retrieve_dataset_files` with the path to this pickle file (it will be
        returned).

    outdir : str or None
        If this is provided, sets the output directory of the downloaded dataset
        files. If None, they will be downloaded to the current directory.

    maxtimeout : float
        The maximum time in seconds to wait for the LCC-Server to respond with a
        result before timing out. You can use the `retrieve_dataset_files`
        function to get results later as needed.

    refresh : float
        The time to wait in seconds before pinging the LCC-Server to see if a
        search query has completed and dataset result files can be downloaded.

    Returns
    -------

    tuple
        Returns a tuple with the following elements::

            (search result status dict,
             search result CSV file path,
             search result LC ZIP path)

    '''

    # turn the input into a param dict

    coords = '%.5f %.5f %.1f' % (center_ra, center_decl, radiusarcmin)
    params = {
        'coords':coords
    }

    if collections:
        params['collections'] = collections
    if columns:
        params['columns'] = columns
    if filters:
        params['filters'] = filters
    if sortspec:
        params['sortspec'] = json.dumps([sortspec])
    if samplespec:
        params['samplespec'] = int(samplespec)
    if limitspec:
        params['limitspec'] = int(limitspec)

    params['visibility'] = result_visibility
    params['emailwhendone'] = email_when_done

    # we won't wait for the LC ZIP to complete if email_when_done = True
    if email_when_done:
        download_data = False

    # check if we have an API key already
    have_apikey, apikey, expires = check_existing_apikey(lcc_server)

    # if not, get a new one
    if not have_apikey:
        apikey, expires = get_new_apikey(lcc_server)

    # hit the server
    api_url = '%s/api/conesearch' % lcc_server

    searchresult = submit_post_searchquery(api_url, params, apikey)

    # check the status of the search
    status = searchresult[0]

    # now we'll check if we want to download the data
    if download_data:

        if status == 'ok':

            LOGINFO('query complete, downloading associated data...')
            csv, lczip, pkl = retrieve_dataset_files(searchresult,
                                                     outdir=outdir,
                                                     apikey=apikey)

            if pkl:
                return searchresult[1], csv, lczip, pkl
            else:
                return searchresult[1], csv, lczip

        elif status == 'background':

            LOGINFO('query is not yet complete, '
                    'waiting up to %.1f minutes, '
                    'updates every %s seconds (hit Ctrl+C to cancel)...' %
                    (maxtimeout/60.0, refresh))

            timewaited = 0.0

            while timewaited < maxtimeout:

                try:

                    time.sleep(refresh)
                    csv, lczip, pkl = retrieve_dataset_files(searchresult,
                                                             outdir=outdir,
                                                             apikey=apikey)

                    if (csv and os.path.exists(csv) and
                        lczip and os.path.exists(lczip)):

                        LOGINFO('all dataset products collected')
                        return searchresult[1], csv, lczip

                    timewaited = timewaited + refresh

                except KeyboardInterrupt:

                    LOGWARNING('abandoned wait for downloading data')
                    return searchresult[1], None, None

            LOGERROR('wait timed out.')
            return searchresult[1], None, None

        else:

            LOGERROR('could not download the data for this query result')
            return searchresult[1], None, None

    else:

        return searchresult[1], None, None


def xmatch_search(lcc_server,
                  file_to_upload,
                  xmatch_dist_arcsec=3.0,
                  result_visibility='unlisted',
                  email_when_done=False,
                  collections=None,
                  columns=None,
                  filters=None,
                  sortspec=None,
                  limitspec=None,
                  samplespec=None,
                  download_data=True,
                  outdir=None,
                  maxtimeout=300.0,
                  refresh=15.0):

    '''This runs a cross-match search query.

    Parameters
    ----------

    lcc_server : str
        This is the base URL of the LCC-Server to talk to.  (e.g. for HAT, use:
        https://data.hatsurveys.org)

    file_to_upload : str
        This is the path to a text file containing objectid, RA, declination
        rows for the objects to cross-match against the LCC-Server
        collections. This should follow the format of the following example::

            # example object and coordinate list
            # objectid ra dec
            aaa 289.99698 44.99839
            bbb 293.358 -23.206
            ccc 294.197 +23.181
            ddd 19 25 27.9129 +42 47 03.693
            eee 19:25:27 -42:47:03.21
            # .
            # .
            # .
            # etc. lines starting with '#' will be ignored
            # (max 5000 objects)

    xmatch_dist_arcsec : float
        This is the maximum distance in arcseconds to consider when
        cross-matching objects in the uploaded file to the LCC-Server's
        collections. The maximum allowed distance is 30 arcseconds. Multiple
        matches to an uploaded object are possible and will be returned in order
        of increasing distance grouped by input `objectid`.

    result_visibility : {'private', 'unlisted', 'public'}
        This sets the visibility of the dataset produced from the search
        result::

               'private' -> the dataset and its products are not visible or
                            accessible by any user other than the one that
                            created the dataset.

               'unlisted' -> the dataset and its products are not visible in the
                             list of public datasets, but can be accessed if the
                             dataset URL is known

               'public' -> the dataset and its products are visible in the list
                           of public datasets and can be accessed by anyone.

    email_when_done : bool
        If True, the LCC-Server will email you when the search is complete. This
        will also set `download_data` to False. Using this requires an
        LCC-Server account and an API key tied to that account.

    collections : list of str or None
        This is a list of LC collections to search in. If this is None, all
        collections will be searched.

    columns : list of str or None
        This is a list of columns to return in the results. Matching objects'
        object IDs, RAs, DECs, and links to light curve files will always be
        returned so there is no need to specify these columns. If None, only
        these columns will be returned: 'objectid', 'ra', 'decl', 'lcfname'

    filters : str or None
        This is an SQL-like string to use to filter on database columns in the
        LCC-Server's collections. To see the columns available for a search,
        visit the Collections tab in the LCC-Server's browser UI. The filter
        operators allowed are::

            lt      -> less than
            gt      -> greater than
            ge      -> greater than or equal to
            le      -> less than or equal to
            eq      -> equal to
            ne      -> not equal to
            ct      -> contains text
            isnull  -> column value is null
            notnull -> column value is not null

        You may use the `and` and `or` operators between filter specifications
        to chain them together logically.

        Example filter strings::

            "(propermotion gt 200.0) and (sdssr lt 11.0)"
            "(dered_jmag_kmag gt 2.0) and (aep_000_stetsonj gt 10.0)"
            "(gaia_status ct 'ok') and (propermotion gt 300.0)"
            "(simbad_best_objtype ct 'RR') and (dered_sdssu_sdssg lt 0.5)"

    sortspec : tuple of two strs or None
        If not None, this should be a tuple of two items::

            ('column to sort by', 'asc|desc')

        This sets the column to sort the results by. For cone_search, the
        default column and sort order are 'dist_arcsec' and 'asc', meaning the
        distance from the search center in ascending order.

    samplespec : int or None
        If this is an int, will indicate how many rows from the initial search
        result will be uniformly random sampled and returned.

    limitspec : int or None
        If this is an int, will indicate how many rows from the initial search
        result to return in total.

        `sortspec`, `samplespec`, and `limitspec` are applied in this order:

            sample -> sort -> limit

    download_data : bool
        This sets if the accompanying data from the search results will be
        downloaded automatically. This includes the data table CSV, the dataset
        pickle file, and a light curve ZIP file. Note that if the search service
        indicates that your query is still in progress, this function will block
        until the light curve ZIP file becomes available. The maximum wait time
        in seconds is set by maxtimeout and the refresh interval is set by
        refresh.

        To avoid the wait block, set download_data to False and the function
        will write a pickle file to `~/.astrobase/lccs/query-[setid].pkl`
        containing all the information necessary to retrieve these data files
        later when the query is done. To do so, call the
        `retrieve_dataset_files` with the path to this pickle file (it will be
        returned).

    outdir : str or None
        If this is provided, sets the output directory of the downloaded dataset
        files. If None, they will be downloaded to the current directory.

    maxtimeout : float
        The maximum time in seconds to wait for the LCC-Server to respond with a
        result before timing out. You can use the `retrieve_dataset_files`
        function to get results later as needed.

    refresh : float
        The time to wait in seconds before pinging the LCC-Server to see if a
        search query has completed and dataset result files can be downloaded.

    Returns
    -------

    tuple
        Returns a tuple with the following elements::

            (search result status dict,
             search result CSV file path,
             search result LC ZIP path)

    '''

    with open(file_to_upload) as infd:
        xmq = infd.read()

    # check the number of lines in the input
    xmqlines = len(xmq.split('\n')[:-1])

    if xmqlines > 5000:

        LOGERROR('you have more than 5000 lines in the file to upload: %s' %
                 file_to_upload)
        return None, None, None

    # turn the input into a param dict
    params = {'xmq':xmq,
              'xmd':xmatch_dist_arcsec}

    if collections:
        params['collections'] = collections
    if columns:
        params['columns'] = columns
    if filters:
        params['filters'] = filters

    if sortspec:
        params['sortspec'] = json.dumps([sortspec])
    if samplespec:
        params['samplespec'] = int(samplespec)
    if limitspec:
        params['limitspec'] = int(limitspec)

    params['visibility'] = result_visibility
    params['emailwhendone'] = email_when_done

    # we won't wait for the LC ZIP to complete if email_when_done = True
    if email_when_done:
        download_data = False

    # check if we have an API key already
    have_apikey, apikey, expires = check_existing_apikey(lcc_server)

    # if not, get a new one
    if not have_apikey:
        apikey, expires = get_new_apikey(lcc_server)

    # hit the server
    api_url = '%s/api/xmatch' % lcc_server

    searchresult = submit_post_searchquery(api_url, params, apikey)

    # check the status of the search
    status = searchresult[0]

    # now we'll check if we want to download the data
    if download_data:

        if status == 'ok':

            LOGINFO('query complete, downloading associated data...')
            csv, lczip, pkl = retrieve_dataset_files(searchresult,
                                                     outdir=outdir,
                                                     apikey=apikey)

            if pkl:
                return searchresult[1], csv, lczip, pkl
            else:
                return searchresult[1], csv, lczip

        elif status == 'background':

            LOGINFO('query is not yet complete, '
                    'waiting up to %.1f minutes, '
                    'updates every %s seconds (hit Ctrl+C to cancel)...' %
                    (maxtimeout/60.0, refresh))

            timewaited = 0.0

            while timewaited < maxtimeout:

                try:

                    time.sleep(refresh)
                    csv, lczip, pkl = retrieve_dataset_files(searchresult,
                                                             outdir=outdir,
                                                             apikey=apikey)

                    if (csv and os.path.exists(csv) and
                        lczip and os.path.exists(lczip)):

                        LOGINFO('all dataset products collected')
                        return searchresult[1], csv, lczip

                    timewaited = timewaited + refresh

                except KeyboardInterrupt:

                    LOGWARNING('abandoned wait for downloading data')
                    return searchresult[1], None, None

            LOGERROR('wait timed out.')
            return searchresult[1], None, None

        else:

            LOGERROR('could not download the data for this query result')
            return searchresult[1], None, None

    else:

        return searchresult[1], None, None

This runs a cross - match search query .


def get_dataset(lcc_server,
                dataset_id,
                strformat=False,
                page=1):
    '''This downloads a JSON form of a dataset from the specified lcc_server.

    If the dataset contains more than 1000 rows, it will be paginated, so you
    must use the `page` kwarg to get the page you want. The dataset JSON will
    contain the keys 'npages', 'currpage', and 'rows_per_page' to help with
    this. The 'rows' key contains the actual data rows as a list of tuples.

    The JSON contains metadata about the query that produced the dataset,
    information about the data table's columns, and links to download the
    dataset's products including the light curve ZIP and the dataset CSV.

    Parameters
    ----------

    lcc_server : str
        This is the base URL of the LCC-Server to talk to.

    dataset_id : str
        This is the unique setid of the dataset you want to get. In the results
        from the `*_search` functions above, this is the value of the
        `infodict['result']['setid']` key in the first item (the infodict) in
        the returned tuple.

    strformat : bool
        This sets if you want the returned data rows to be formatted in their
        string representations already. This can be useful if you're piping the
        returned JSON straight into some sort of UI and you don't want to deal
        with formatting floats, etc. To do this manually when strformat is set
        to False, look at the `coldesc` item in the returned dict, which gives
        the Python and Numpy string format specifiers for each column in the
        data table.

    page : int
        This sets which page of the dataset should be retrieved.

    Returns
    -------

    dict
        This returns the dataset JSON loaded into a dict.

    '''

    urlparams = {'strformat':1 if strformat else 0,
                 'page':page,
                 'json':1}
    urlqs = urlencode(urlparams)

    dataset_url = '%s/set/%s?%s' % (lcc_server, dataset_id, urlqs)

    LOGINFO('retrieving dataset %s from %s, using URL: %s ...' % (lcc_server,
                                                                  dataset_id,
                                                                  dataset_url))

    try:

        # check if we have an API key already
        have_apikey, apikey, expires = check_existing_apikey(lcc_server)

        # if not, get a new one
        if not have_apikey:
            apikey, expires = get_new_apikey(lcc_server)

        # if apikey is not None, add it in as an Authorization: Bearer [apikey]
        # header
        if apikey:
            headers = {'Authorization':'Bearer: %s' % apikey}
        else:
            headers = {}

        # hit the server
        req = Request(dataset_url, data=None, headers=headers)
        resp = urlopen(req)
        dataset = json.loads(resp.read())
        return dataset

    except Exception as e:

        LOGEXCEPTION('could not retrieve the dataset JSON!')
        return None

This downloads a JSON form of a dataset from the specified lcc_server .


def object_info(lcc_server, objectid, db_collection_id):
    '''This gets information on a single object from the LCC-Server.

    Returns a dict with all of the available information on an object, including
    finding charts, comments, object type and variability tags, and
    period-search results (if available).

    If you have an LCC-Server API key present in `~/.astrobase/lccs/` that is
    associated with an LCC-Server user account, objects that are visible to this
    user will be returned, even if they are not visible to the public. Use this
    to look up objects that have been marked as 'private' or 'shared'.

    NOTE: you can pass the result dict returned by this function directly into
    the `astrobase.checkplot.checkplot_pickle_to_png` function, e.g.::

        astrobase.checkplot.checkplot_pickle_to_png(result_dict,
                                                    'object-%s-info.png' %
                                                    result_dict['objectid'])

    to generate a quick PNG overview of the object information.

    Parameters
    ----------

    lcc_server : str
        This is the base URL of the LCC-Server to talk to.

    objectid : str
        This is the unique database ID of the object to retrieve info for. This
        is always returned as the `db_oid` column in LCC-Server search results.

    db_collection_id : str
        This is the collection ID which will be searched for the object. This is
        always returned as the `collection` column in LCC-Server search results.

    Returns
    -------

    dict
        A dict containing the object info is returned. Some important items in
        the result dict:

        - `objectinfo`: all object magnitude, color, GAIA cross-match, and
          object type information available for this object

        - `objectcomments`: comments on the object's variability if available

        - `varinfo`: variability comments, variability features, type tags,
          period and epoch information if available

        - `neighbors`: information on the neighboring objects of this object in
          its parent light curve collection

        - `xmatch`: information on any cross-matches to external catalogs
          (e.g. KIC, EPIC, TIC, APOGEE, etc.)

        - `finderchart`: a base-64 encoded PNG image of the object's DSS2 RED
          finder chart. To convert this to an actual PNG, try the function:
          `astrobase.checkplot.pkl_io._b64_to_file`.

        - `magseries`: a base-64 encoded PNG image of the object's light
          curve. To convert this to an actual PNG, try the function:
          `astrobase.checkplot.pkl_io._b64_to_file`.

        - `pfmethods`: a list of period-finding methods applied to the object if
          any. If this list is present, use the keys in it to get to the actual
          period-finding results for each method. These will contain base-64
          encoded PNGs of the periodogram and phased light curves using the best
          three peaks in the periodogram, as well as period and epoch
          information.

    '''

    urlparams = {
        'objectid':objectid,
        'collection':db_collection_id
    }

    urlqs = urlencode(urlparams)
    url = '%s/api/object?%s' % (lcc_server, urlqs)

    try:

        LOGINFO(
            'getting info for %s in collection %s from %s' % (
                objectid,
                db_collection_id,
                lcc_server
            )
        )

        # check if we have an API key already
        have_apikey, apikey, expires = check_existing_apikey(lcc_server)

        # if not, get a new one
        if not have_apikey:
            apikey, expires = get_new_apikey(lcc_server)

        # if apikey is not None, add it in as an Authorization: Bearer [apikey]
        # header
        if apikey:
            headers = {'Authorization':'Bearer: %s' % apikey}
        else:
            headers = {}

        # hit the server
        req = Request(url, data=None, headers=headers)
        resp = urlopen(req)
        objectinfo = json.loads(resp.read())['result']
        return objectinfo

    except HTTPError as e:

        if e.code == 404:

            LOGERROR(
                'additional info for object %s not '
                'found in collection: %s' % (objectid,
                                             db_collection_id)
            )

        else:

            LOGERROR('could not retrieve object info, '
                     'URL used: %s, error code: %s, reason: %s' %
                     (url, e.code, e.reason))

        return None

This gets information on a single object from the LCC - Server .


def list_recent_datasets(lcc_server, nrecent=25):
    '''This lists recent publicly visible datasets available on the LCC-Server.

    If you have an LCC-Server API key present in `~/.astrobase/lccs/` that is
    associated with an LCC-Server user account, datasets that belong to this
    user will be returned as well, even if they are not visible to the public.

    Parameters
    ----------

    lcc_server : str
        This is the base URL of the LCC-Server to talk to.

    nrecent : int
        This indicates how many recent public datasets you want to list. This is
        always capped at 1000.

    Returns
    -------

    list of dicts
        Returns a list of dicts, with each dict containing info on each dataset.

    '''

    urlparams = {'nsets':nrecent}
    urlqs = urlencode(urlparams)

    url = '%s/api/datasets?%s' % (lcc_server, urlqs)

    try:

        LOGINFO(
            'getting list of recent publicly '
            'visible and owned datasets from %s' % (
                lcc_server,
            )
        )

        # check if we have an API key already
        have_apikey, apikey, expires = check_existing_apikey(lcc_server)

        # if not, get a new one
        if not have_apikey:
            apikey, expires = get_new_apikey(lcc_server)

        # if apikey is not None, add it in as an Authorization: Bearer [apikey]
        # header
        if apikey:
            headers = {'Authorization':'Bearer: %s' % apikey}
        else:
            headers = {}

        # hit the server
        req = Request(url, data=None, headers=headers)
        resp = urlopen(req)
        recent_datasets = json.loads(resp.read())['result']
        return recent_datasets

    except HTTPError as e:

        LOGERROR('could not retrieve recent datasets list, '
                 'URL used: %s, error code: %s, reason: %s' %
                 (url, e.code, e.reason))

        return None

This lists recent publicly visible datasets available on the LCC - Server .


def list_lc_collections(lcc_server):
    '''This lists all light curve collections made available on the LCC-Server.

    If you have an LCC-Server API key present in `~/.astrobase/lccs/` that is
    associated with an LCC-Server user account, light curve collections visible
    to this user will be returned as well, even if they are not visible to the
    public.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server to talk to.

    Returns
    -------

    dict
        Returns a dict containing lists of info items per collection. This
        includes collection_ids, lists of columns, lists of indexed columns,
        lists of full-text indexed columns, detailed column descriptions, number
        of objects in each collection, collection sky coverage, etc.

    '''

    url = '%s/api/collections' % lcc_server

    try:

        LOGINFO(
            'getting list of recent publicly visible '
            'and owned LC collections from %s' % (
                lcc_server,
            )
        )

        # check if we have an API key already
        have_apikey, apikey, expires = check_existing_apikey(lcc_server)

        # if not, get a new one
        if not have_apikey:
            apikey, expires = get_new_apikey(lcc_server)

        # if apikey is not None, add it in as an Authorization: Bearer [apikey]
        # header
        if apikey:
            headers = {'Authorization':'Bearer: %s' % apikey}
        else:
            headers = {}

        # hit the server
        req = Request(url, data=None, headers=headers)
        resp = urlopen(req)
        lcc_list = json.loads(resp.read())['result']['collections']
        return lcc_list

    except HTTPError as e:

        LOGERROR('could not retrieve list of collections, '
                 'URL used: %s, error code: %s, reason: %s' %
                 (url, e.code, e.reason))

        return None

This lists all light curve collections made available on the LCC - Server .


def stetson_jindex(ftimes, fmags, ferrs, weightbytimediff=False):
    '''This calculates the Stetson index for the magseries, based on consecutive
    pairs of observations.

    Based on Nicole Loncke's work for her Planets and Life certificate at
    Princeton in 2014.

    Parameters
    ----------

    ftimes,fmags,ferrs : np.array
        The input mag/flux time-series with all non-finite elements removed.

    weightbytimediff : bool
        If this is True, the Stetson index for any pair of mags will be
        reweighted by the difference in times between them using the scheme in
        Fruth+ 2012 and Zhange+ 2003 (as seen in Sokolovsky+ 2017)::

            w_i = exp(- (t_i+1 - t_i)/ delta_t )

    Returns
    -------

    float
        The calculated Stetson J variability index.

    '''

    ndet = len(fmags)

    if ndet > 9:

        # get the median and ndet
        medmag = npmedian(fmags)

        # get the stetson index elements
        delta_prefactor = (ndet/(ndet - 1))
        sigma_i = delta_prefactor*(fmags - medmag)/ferrs

        # Nicole's clever trick to advance indices by 1 and do x_i*x_(i+1)
        sigma_j = nproll(sigma_i,1)

        if weightbytimediff:

            difft = npdiff(ftimes)
            deltat = npmedian(difft)

            weights_i = npexp(- difft/deltat )
            products = (weights_i*sigma_i[1:]*sigma_j[1:])
        else:
            # ignore first elem since it's actually x_0*x_n
            products = (sigma_i*sigma_j)[1:]

        stetsonj = (
            npsum(npsign(products) * npsqrt(npabs(products)))
        ) / ndet

        return stetsonj

    else:

        LOGERROR('not enough detections in this magseries '
                 'to calculate stetson J index')
        return npnan

This calculates the Stetson index for the magseries based on consecutive pairs of observations .


def lightcurve_moments(ftimes, fmags, ferrs):
    '''This calculates the weighted mean, stdev, median, MAD, percentiles, skew,
    kurtosis, fraction of LC beyond 1-stdev, and IQR.

    Parameters
    ----------

    ftimes,fmags,ferrs : np.array
        The input mag/flux time-series with all non-finite elements removed.

    Returns
    -------

    dict
        A dict with all of the light curve moments calculated.

    '''

    ndet = len(fmags)

    if ndet > 9:

        # now calculate the various things we need
        series_median = npmedian(fmags)
        series_wmean = (
            npsum(fmags*(1.0/(ferrs*ferrs)))/npsum(1.0/(ferrs*ferrs))
        )
        series_mad = npmedian(npabs(fmags - series_median))
        series_stdev = 1.483*series_mad
        series_skew = spskew(fmags)
        series_kurtosis = spkurtosis(fmags)

        # get the beyond1std fraction
        series_above1std = len(fmags[fmags > (series_median + series_stdev)])
        series_below1std = len(fmags[fmags < (series_median - series_stdev)])

        # this is the fraction beyond 1 stdev
        series_beyond1std = (series_above1std + series_below1std)/float(ndet)

        # get the magnitude percentiles
        series_mag_percentiles = nppercentile(
            fmags,
            [5.0,10,17.5,25,32.5,40,60,67.5,75,82.5,90,95]
        )

        return {
            'median':series_median,
            'wmean':series_wmean,
            'mad':series_mad,
            'stdev':series_stdev,
            'skew':series_skew,
            'kurtosis':series_kurtosis,
            'beyond1std':series_beyond1std,
            'mag_percentiles':series_mag_percentiles,
            'mag_iqr': series_mag_percentiles[8] - series_mag_percentiles[3],
        }

    else:
        LOGERROR('not enough detections in this magseries '
                 'to calculate light curve moments')
        return None

This calculates the weighted mean stdev median MAD percentiles skew kurtosis fraction of LC beyond 1 - stdev and IQR .


def lightcurve_flux_measures(ftimes, fmags, ferrs, magsarefluxes=False):
    '''This calculates percentiles and percentile ratios of the flux.

    Parameters
    ----------

    ftimes,fmags,ferrs : np.array
        The input mag/flux time-series with all non-finite elements removed.

    magsarefluxes : bool
        If the `fmags` array actually contains fluxes, will not convert `mags`
        to fluxes before calculating the percentiles.

    Returns
    -------

    dict
        A dict with all of the light curve flux percentiles and percentile
        ratios calculated.

    '''

    ndet = len(fmags)

    if ndet > 9:

        # get the fluxes
        if magsarefluxes:
            series_fluxes = fmags
        else:
            series_fluxes = 10.0**(-0.4*fmags)

        series_flux_median = npmedian(series_fluxes)

        # get the percent_amplitude for the fluxes
        series_flux_percent_amplitude = (
            npmax(npabs(series_fluxes))/series_flux_median
        )

        # get the flux percentiles
        series_flux_percentiles = nppercentile(
            series_fluxes,
            [5.0,10,17.5,25,32.5,40,60,67.5,75,82.5,90,95]
        )
        series_frat_595 = (
            series_flux_percentiles[-1] - series_flux_percentiles[0]
        )
        series_frat_1090 = (
            series_flux_percentiles[-2] - series_flux_percentiles[1]
        )
        series_frat_175825 = (
            series_flux_percentiles[-3] - series_flux_percentiles[2]
        )
        series_frat_2575 = (
            series_flux_percentiles[-4] - series_flux_percentiles[3]
        )
        series_frat_325675 = (
            series_flux_percentiles[-5] - series_flux_percentiles[4]
        )
        series_frat_4060 = (
            series_flux_percentiles[-6] - series_flux_percentiles[5]
        )

        # calculate the flux percentile ratios
        series_flux_percentile_ratio_mid20 = series_frat_4060/series_frat_595
        series_flux_percentile_ratio_mid35 = series_frat_325675/series_frat_595
        series_flux_percentile_ratio_mid50 = series_frat_2575/series_frat_595
        series_flux_percentile_ratio_mid65 = series_frat_175825/series_frat_595
        series_flux_percentile_ratio_mid80 = series_frat_1090/series_frat_595

        # calculate the ratio of F595/median flux
        series_percent_difference_flux_percentile = (
            series_frat_595/series_flux_median
        )
        series_percentile_magdiff = -2.5*nplog10(
            series_percent_difference_flux_percentile
        )

        return {
            'flux_median':series_flux_median,
            'flux_percent_amplitude':series_flux_percent_amplitude,
            'flux_percentiles':series_flux_percentiles,
            'flux_percentile_ratio_mid20':series_flux_percentile_ratio_mid20,
            'flux_percentile_ratio_mid35':series_flux_percentile_ratio_mid35,
            'flux_percentile_ratio_mid50':series_flux_percentile_ratio_mid50,
            'flux_percentile_ratio_mid65':series_flux_percentile_ratio_mid65,
            'flux_percentile_ratio_mid80':series_flux_percentile_ratio_mid80,
            'percent_difference_flux_percentile':series_percentile_magdiff,
        }

    else:

        LOGERROR('not enough detections in this magseries '
                 'to calculate flux measures')
        return None

This calculates percentiles and percentile ratios of the flux .


def all_nonperiodic_features(times, mags, errs,
                             magsarefluxes=False,
                             stetson_weightbytimediff=True):
    '''This rolls up the feature functions above and returns a single dict.

    NOTE: this doesn't calculate the CDPP to save time since binning and
    smoothing takes a while for dense light curves.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to calculate CDPP for.

    magsarefluxes : bool
        If True, indicates `mags` is actually an array of flux values.

    stetson_weightbytimediff : bool
        If this is True, the Stetson index for any pair of mags will be
        reweighted by the difference in times between them using the scheme in
        Fruth+ 2012 and Zhange+ 2003 (as seen in Sokolovsky+ 2017)::

            w_i = exp(- (t_i+1 - t_i)/ delta_t )

    Returns
    -------

    dict
        Returns a dict with all of the variability features.

    '''

    # remove nans first
    finiteind = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
    ftimes, fmags, ferrs = times[finiteind], mags[finiteind], errs[finiteind]

    # remove zero errors
    nzind = npnonzero(ferrs)
    ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]

    xfeatures = nonperiodic_lightcurve_features(times, mags, errs,
                                                magsarefluxes=magsarefluxes)
    stetj = stetson_jindex(ftimes, fmags, ferrs,
                           weightbytimediff=stetson_weightbytimediff)
    stetk = stetson_kindex(fmags, ferrs)

    xfeatures.update({'stetsonj':stetj,
                      'stetsonk':stetk})

    return xfeatures

def
all_nonperiodic_features
(
times
,
mags
,
errs
,
magsarefluxes
=
False
,
stetson_weightbytimediff
=
True
)
:
# remove nans first
finiteind
=
npisfinite
(
times
)
&
npisfinite
(
mags
)
&
npisfinite
(
errs
)
ftimes
,
fmags
,
ferrs
=
times
[
finiteind
]
,
mags
[
finiteind
]
,
errs
[
finiteind
]
# remove zero errors
nzind
=
npnonzero
(
ferrs
)
ftimes
,
fmags
,
ferrs
=
ftimes
[
nzind
]
,
fmags
[
nzind
]
,
ferrs
[
nzind
]
xfeatures
=
nonperiodic_lightcurve_features
(
times
,
mags
,
errs
,
magsarefluxes
=
magsarefluxes
)
stetj
=
stetson_jindex
(
ftimes
,
fmags
,
ferrs
,
weightbytimediff
=
stetson_weightbytimediff
)
stetk
=
stetson_kindex
(
fmags
,
ferrs
)
xfeatures
.
update
(
{
'stetsonj'
:
stetj
,
'stetsonk'
:
stetk
}
)
return
xfeatures

This rolls up the feature functions above and returns a single dict .


def _bls_runner(times,
                mags,
                nfreq,
                freqmin,
                stepsize,
                nbins,
                minduration,
                maxduration):
    '''This runs the pyeebls.eebls function using the given inputs.

    Parameters
    ----------

    times,mags : np.array
        The input magnitude time-series to search for transits.

    nfreq : int
        The number of frequencies to use when searching for transits.

    freqmin : float
        The minimum frequency of the period-search -> max period that will be
        used for the search.

    stepsize : float
        The step-size in frequency to use to generate a frequency-grid.

    nbins : int
        The number of phase bins to use.

    minduration : float
        The minimum fractional transit duration that will be considered.

    maxduration : float
        The maximum fractional transit duration that will be considered.

    Returns
    -------

    dict
        Returns a dict of the form::

            {
                'power':           the periodogram power array,
                'bestperiod':      the best period found,
                'bestpower':       the highest peak of the periodogram power,
                'transdepth':      transit depth found by eebls.f,
                'transduration':   transit duration found by eebls.f,
                'transingressbin': transit ingress bin found by eebls.f,
                'transegressbin':  transit egress bin found by eebls.f,
            }

    '''

    workarr_u = npones(times.size)
    workarr_v = npones(times.size)

    blsresult = eebls(times, mags,
                      workarr_u, workarr_v,
                      nfreq, freqmin, stepsize,
                      nbins, minduration, maxduration)

    return {'power':blsresult[0],
            'bestperiod':blsresult[1],
            'bestpower':blsresult[2],
            'transdepth':blsresult[3],
            'transduration':blsresult[4],
            'transingressbin':blsresult[5],
            'transegressbin':blsresult[6]}

def _bls_runner ( times , mags , nfreq , freqmin , stepsize , nbins , minduration , maxduration ) : workarr_u = npones ( times . size ) workarr_v = npones ( times . size ) blsresult = eebls ( times , mags , workarr_u , workarr_v , nfreq , freqmin , stepsize , nbins , minduration , maxduration ) return { 'power' : blsresult [ 0 ] , 'bestperiod' : blsresult [ 1 ] , 'bestpower' : blsresult [ 2 ] , 'transdepth' : blsresult [ 3 ] , 'transduration' : blsresult [ 4 ] , 'transingressbin' : blsresult [ 5 ] , 'transegressbin' : blsresult [ 6 ] }

This runs the pyeebls . eebls function using the given inputs .


def _parallel_bls_worker(task):
    '''
    This wraps the BLS function for the parallel driver below.

    Parameters
    ----------

    tasks : tuple
        This is of the form::

            task[0] = times
            task[1] = mags
            task[2] = nfreq
            task[3] = freqmin
            task[4] = stepsize
            task[5] = nbins
            task[6] = minduration
            task[7] = maxduration

    Returns
    -------

    dict
        Returns a dict of the form::

            {
                'power':           the periodogram power array,
                'bestperiod':      the best period found,
                'bestpower':       the highest peak of the periodogram power,
                'transdepth':      transit depth found by eebls.f,
                'transduration':   transit duration found by eebls.f,
                'transingressbin': transit ingress bin found by eebls.f,
                'transegressbin':  transit egress bin found by eebls.f,
            }

    '''

    try:

        return _bls_runner(*task)

    except Exception as e:

        LOGEXCEPTION('BLS failed for task %s' % repr(task[2:]))

        return {
            'power':nparray([npnan for x in range(task[2])]),
            'bestperiod':npnan,
            'bestpower':npnan,
            'transdepth':npnan,
            'transduration':npnan,
            'transingressbin':npnan,
            'transegressbin':npnan
        }

def
_parallel_bls_worker
(
task
)
:
try
:
return
_bls_runner
(
*
task
)
except
Exception
as
e
:
LOGEXCEPTION
(
'BLS failed for task %s'
%
repr
(
task
[
2
:
]
)
)
return
{
'power'
:
nparray
(
[
npnan
for
x
in
range
(
task
[
2
]
)
]
)
,
'bestperiod'
:
npnan
,
'bestpower'
:
npnan
,
'transdepth'
:
npnan
,
'transduration'
:
npnan
,
'transingressbin'
:
npnan
,
'transegressbin'
:
npnan
}

This wraps the BLS function for the parallel driver below .