def apogee_default_dr(dr=None):
    """
    Check if dr argument is provided, if none then use default

    :param dr: APOGEE DR
    :type dr: int
    :return: APOGEE DR
    :rtype: int
    :History:
        | 2017-Oct-26 - Written - Henry Leung (University of Toronto)
        | 2018-Sept-08 - Updated - Henry Leung (University of Toronto)
    """
    if dr == 15:
        print("SDSS APOGEE DR15 is equivalent to DR14, so astroNN is using DR14 even you set DR15")
        dr = 14

    if dr is None:
        try:
            redux_ver = os.environ["RESULTS_VERS"]  # RESULTS_VERS is from Jo Bovy APOGEE Tool
            if redux_ver == "v402":
                dr = 11
            elif redux_ver == "v603":
                dr = 12
            elif redux_ver == "l30e.2":
                dr = 13
            elif redux_ver == "l31c.2":
                dr = 14
            elif redux_ver == "l33":
                dr = 16
        except KeyError:
            pass

        if dr is None:  # if it is still None
            dr = 16
            print(f'dr is not provided, using default dr={dr}')
    else:
        pass

    return dr def aspcap_mask(elem, dr=None):
    """
    | To load ASPCAP elements window masks
    | DR14 Elements: ``'C', 'CI', 'N', 'O', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'K', 'Ca', 'TI', 'TiII', 'V', 'Cr', 'Mn',
                     'Fe', 'Co', 'Ni', 'Cu', 'Ge', 'Ce', 'Rb', 'Y', 'Nd'``

    :param elem: element name
    :type elem: str
    :param dr: apogee dr
    :type dr: int
    :return: mask
    :rtype: ndarray[bool]
    :History: 2018-Mar-24 - Written - Henry Leung (University of Toronto)
    """
    if elem.lower() == 'c1':
        elem = 'CI'

    elif elem.lower() == 'ti2':
        elem = 'TiII'

    dr = apogee_default_dr(dr=dr)

    if 14 <= dr <= 16:
        aspcap_code = 'l31c'
        elem_list = ['C', 'CI', 'N', 'O', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'K', 'Ca', 'TI', 'TiII', 'V', 'Cr', 'Mn',
                     'Fe', 'Co', 'Ni', 'Cu', 'Ge', 'Ce', 'Rb', 'Y', 'Nd']
    else:
        raise ValueError('Only DR14-DR16 is supported currently')

    masks = np.load(
        os.path.join(astroNN.data.datapath(), f'aspcap_{aspcap_code}_masks.npy'))

    try:
        # turn everything to lowercase to avoid case-related issue
        index = [x.lower() for x in elem_list].index(elem.lower())
    except ValueError:
        # nicely handle if element not found
        print(f'Element not found, the only elements for dr{dr} supported are {elem_list}')
        return None

    return [(masks & 2 ** index) != 0][0] def bitmask_boolean(bitmask, target_bit):
    """
    Turn bitmask to boolean with provided bitmask array and target bit to mask

    :param bitmask: bitmask
    :type bitmask: ndarray
    :param target_bit: target bit to mask
    :type target_bit: list[int]
    :return: boolean array, True for clean, False for masked
    :rtype: ndarray[bool]
    :History: 2018-Feb-03 - Written - Henry Leung (University of Toronto)
    """
    target_bit = np.array(target_bit)
    target_bit = np.sum(2 ** target_bit)
    bitmask = np.atleast_2d(bitmask)
    boolean_output = np.zeros(bitmask.shape, dtype=bool)
    boolean_output[(bitmask & target_bit) != 0] = True
    return boolean_output def bitmask_decompositor(bit):
    """
    To decompose a bit from bitmask array to individual bit

    :param bit: bitmask
    :type bit: int
    :return: boolean array, True for clean, False for masked
    :rtype: ndarray[bool]
    :History: 2018-Feb-03 - Written - Henry Leung (University of Toronto)
    """
    bitmask_num = int(bit)
    if bitmask_num < 0:
        raise ValueError(f"Your number ({bit}) is not valid, this value must not from a bitmask")
    if bitmask_num == 0:
        print('0 corresponds to good pixel, thus this bit cannot be decomposed')
        return None
    decomposited_bits = [int(np.log2(bitmask_num))]
    while True:
        if bitmask_num - 2 ** decomposited_bits[-1] == 0:
            decomposited_bits.sort()
            decomposited_bits = np.array(decomposited_bits)
            break
        bitmask_num -= 2 ** decomposited_bits[-1]
        if bitmask_num != 0:
            decomposited_bits.append(int(np.log2(bitmask_num)))

    return decomposited_bits def chips_split(spectra, dr=None):
    """
    To split APOGEE spectra into RGB chips, will delete the gap if detected

    :param spectra: APOGEE spectrum/spectra
    :type spectra: ndarray
    :param dr: data release
    :type dr: Union(int, NoneType)
    :return: 3 ndarrays which are spectra_blue, spectra_green, spectra_red
    :rtype: ndarray
    :History:
        | 2017-Nov-20 - Written - Henry Leung (University of Toronto)
        | 2017-Dec-17 - Updated - Henry Leung (University of Toronto)
    """
    dr = apogee_default_dr(dr=dr)
    info = chips_pix_info(dr=dr)
    blue = info[1] - info[0]
    green = info[3] - info[2]
    red = info[5] - info[4]

    spectra = np.atleast_2d(spectra)

    if spectra.shape[1] == 8575:
        spectra = gap_delete(spectra, dr=dr)
        print("Raw Spectra detected, astroNN has deleted the gap automatically")
    elif spectra.shape[1] == info[6]:
        pass
    else:
        raise EnvironmentError('Are you sure you are giving astroNN APOGEE spectra?')

    spectra_blue = spectra[:, 0:blue]
    spectra_green = spectra[:, blue:(blue + green)]
    spectra_red = spectra[:, (blue + green):(blue + green + red)]

    return spectra_blue, spectra_green, spectra_red def apogee_continuum(spectra, spectra_err, cont_mask=None, deg=2, dr=None, bitmask=None, target_bit=None,
                     mask_value=1.):
    """
    It is designed only for apogee spectra by fitting Chebyshev polynomials to the flux values in the continuum mask 
    by chips. The resulting continuum will have the same shape as `fluxes`.
        
    :param spectra: spectra
    :type spectra: ndarray
    :param spectra_err: spectra uncertainty, same shape as spectra
    :type spectra_err: ndarray
    :param cont_mask: continuum mask
    :type cont_mask: ndarray[bool]
    :param deg: The degree of Chebyshev polynomial to use in each region, default is 2 which works the best so far
    :type deg: int
    :param dr: apogee dr
    :type dr: int
    :param bitmask: bitmask array of the spectra, same shape as spectra
    :type bitmask: ndarray
    :param target_bit: a list of bit to be masked
    :type target_bit: Union(int, list[int], ndarray[int])
    :param mask_value: if a pixel is determined to be a bad pixel, this value will be used to replace that pixel flux
    :type mask_value: Union(int, float)
    :return: normalized spectra, normalized spectra uncertainty
    :rtype: ndarray, ndarray
    :History: 2018-Mar-21 - Written - Henry Leung (University of Toronto)
    """
    dr = apogee_default_dr(dr=dr)

    spectra = gap_delete(spectra, dr=dr)
    flux_errs = gap_delete(spectra_err, dr=dr)

    spectra_blue, spectra_green, spectra_red = chips_split(spectra, dr=dr)
    yerrs_blue, yerrs_green, yerrs_red = chips_split(flux_errs, dr=dr)

    if cont_mask is None:
        maskpath = os.path.join(astroNN.data.datapath(), f'dr{dr}_contmask.npy')
        cont_mask = np.load(maskpath)

    con_mask_blue, con_mask_green, con_mask_red = chips_split(cont_mask, dr=dr)
    con_mask_blue, con_mask_green, con_mask_red = con_mask_blue[0], con_mask_green[0], con_mask_red[0]

    # Continuum chips by chips
    blue_spectra, blue_spectra_err = continuum(spectra_blue, yerrs_blue, cont_mask=con_mask_blue, deg=deg)
    green_spectra, green_spectra_err = continuum(spectra_green, yerrs_green, cont_mask=con_mask_green, deg=deg)
    red_spectra, red_spectra_err = continuum(spectra_red, yerrs_red, cont_mask=con_mask_red, deg=deg)

    normalized_spectra = np.concatenate((blue_spectra, green_spectra, red_spectra), axis=1)
    normalized_spectra_err = np.concatenate((blue_spectra_err, green_spectra_err, red_spectra_err), axis=1)

    # set negative flux and error as 0
    normalized_spectra[normalized_spectra < 0.] = 0.
    normalized_spectra_err[normalized_spectra < 0.] = 0.

    # set inf and nan as 0
    normalized_spectra[np.isinf(normalized_spectra)] = mask_value
    normalized_spectra[np.isnan(normalized_spectra)] = mask_value
    normalized_spectra_err[np.isinf(normalized_spectra)] = 0.
    normalized_spectra_err[np.isnan(normalized_spectra)] = 0.

    if bitmask is not None:
        bitmask = gap_delete(bitmask, dr=dr)
        if target_bit is None:
            target_bit = [0, 1, 2, 3, 4, 5, 6, 7, 12]

        mask = bitmask_boolean(bitmask, target_bit)
        normalized_spectra[mask] = mask_value
        normalized_spectra_err[mask] = mask_value

    return normalized_spectra, normalized_spectra_err def gap_delete(spectra, dr=None):
    """
    To delete the gap between APOGEE CCDs from the original 8575 pixels spectra

    :param spectra: The original 8575 pixels spectrum/spectra
    :type spectra: ndarray
    :param dr: data release
    :type dr: Union(int, NoneType)
    :return: Gap deleted spectrum/spectra
    :rtype: ndarray
    :History:
        | 2017-Oct-26 - Written - Henry Leung (University of Toronto)
        | 2017-Dec-16 - Updated - Henry Leung (University of Toronto)
    """
    dr = apogee_default_dr(dr=dr)
    spectra = np.atleast_2d(spectra)
    info = chips_pix_info(dr=dr)

    if spectra.shape[1] != 8575 and spectra.shape[1] != info[6]:
        raise EnvironmentError('Are you sure you are giving astroNN APOGEE spectra?')
    if spectra.shape[1] != info[6]:
        spectra = spectra[:, np.r_[info[0]:info[1], info[2]:info[3], info[4]:info[5]]]

    return spectra def combined_spectra(
    dr=None,
    location=None,
    field=None,
    apogee=None,
    telescope=None,
    verbose=1,
    flag=None,
):
    """
    Download the required combined spectra file a.k.a aspcapStar

    :param dr: APOGEE DR
    :type dr: int
    :param location: Location ID [Optional]
    :type location: int
    :param field: Field [Optional]
    :type field: str
    :param apogee: Apogee ID
    :type apogee: str
    :param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
    :type telescope: str
    :param verbose: verbose, set 0 to silent most logging
    :type verbose: int
    :param flag: 0: normal, 1: force to re-download
    :type flag: int

    :return: full file path and download in background if not found locally, False if cannot be found on server
    :rtype: str
    :History:
        | 2017-Oct-15 - Written - Henry Leung (University of Toronto)
        | 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
    """
    dr = apogee_default_dr(dr=dr)

    # for DR16=<, location is expected to be none because field is used
    if (location is None and dr < 16) or (
        field is None and dr >= 16
    ):  # try to load info if not enough info
        global _ALLSTAR_TEMP
        if not str(f"dr{dr}") in _ALLSTAR_TEMP:
            _ALLSTAR_TEMP[f"dr{dr}"] = fits.getdata(allstar(dr=dr))
        if telescope is None:
            matched_idx = [
                np.nonzero(_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)[0]
            ][0]
        else:
            matched_idx = [
                np.nonzero(
                    [
                        (_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)
                        & (_ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"] == telescope)
                    ]
                )
            ][0][1]
        if len(matched_idx) == 0:
            raise ValueError(
                f"No entry found in allstar DR{dr} met with your requirement!!"
            )

        location = (
            _ALLSTAR_TEMP[f"dr{dr}"]["LOCATION_ID"][matched_idx][0]
            if not location
            else location
        )
        field = (
            _ALLSTAR_TEMP[f"dr{dr}"]["FIELD"][matched_idx][0] if not field else field
        )
        telescope = (
            _ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"][matched_idx][0]
            if not telescope
            else telescope
        )

    if dr == 13:
        reduce_prefix = "r6"
        aspcap_code = "l30e"
        str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/"

        filename = f"aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits"
        hash_filename = f"stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum"
        urlstr = str1 + filename

        # check folder existence
        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/",
            str(location),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

        fullfilename = os.path.join(fullfoldername, filename)

    elif dr == 14:
        reduce_prefix = "r8"
        aspcap_code = "l31c"
        str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/{location}/"

        filename = f"aspcapStar-{reduce_prefix}-{aspcap_code}.2-{apogee}.fits"
        hash_filename = f"stars_{aspcap_code}_{aspcap_code}.2_{location}.sha1sum"
        urlstr = str1 + filename

        # check folder existence
        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{aspcap_code}/{aspcap_code}.2/",
            str(location),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

        fullfilename = os.path.join(fullfoldername, filename)
    elif dr == 16:
        reduce_prefix = "r12"
        aspcap_code = "l33"
        str1 = f"https://data.sdss.org/sas/dr16/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/"

        filename = f"aspcapStar-{reduce_prefix}-{apogee}.fits"
        hash_filename = f"{reduce_prefix}_{aspcap_code}_{telescope}_{field}.sha1sum"
        urlstr = str1 + filename

        # check folder existence
        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}",
            str(f"{field}"),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

        fullfilename = os.path.join(fullfoldername, filename)
    elif dr == 17:
        reduce_prefix = "dr17"
        aspcap_code = "synspec"
        str1 = f"https://data.sdss.org/sas/dr17/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}/{field}/"

        filename = f"aspcapStar-{reduce_prefix}-{apogee}.fits"
        hash_filename = f"{reduce_prefix}_{aspcap_code}_{telescope}_{field}.sha1sum"
        urlstr = str1 + filename

        # check folder existence
        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/aspcap/{reduce_prefix}/{aspcap_code}/{telescope}",
            str(f"{field}"),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

        fullfilename = os.path.join(fullfoldername, filename)
    else:
        raise ValueError("combined_spectra() only supports APOGEE DR13-DR17")

    # check hash file
    full_hash_filename = os.path.join(fullfoldername, hash_filename)
    if not os.path.isfile(full_hash_filename):
        # return warning flag if the location_id cannot even be found
        try:
            urllib.request.urlopen(str1)
        except urllib.error.HTTPError:
            return warning_flag
        urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)

    hash_list = np.loadtxt(full_hash_filename, dtype="str").T

    # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
    file_hash = hash_list[0][np.argwhere(hash_list[1] == filename)]

    if os.path.isfile(fullfilename) and flag is None:
        checksum = filehash(fullfilename, algorithm="sha1")
        if checksum != file_hash and len(file_hash) != 0:
            warnings.warn(
                "File corruption detected, astroNN is attempting to download again"
            )
            combined_spectra(
                dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1
            )

        if verbose == 1:
            logging.info(fullfilename + " was found!")

    elif not os.path.isfile(fullfilename) or flag == 1:
        try:
            urllib.request.urlretrieve(urlstr, fullfilename)
            logging.info(
                f"Downloaded DR{dr} combined file successfully to {fullfilename}"
            )
            checksum = filehash(fullfilename, algorithm="sha1")
            if checksum != file_hash and len(file_hash) != 0:
                warnings.warn(
                    "File corruption detected, astroNN is attempting to download again"
                )
                combined_spectra(
                    dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1
                )
        except urllib.error.HTTPError as emsg:
            if "401" in str(emsg):
                fullfilename = __apogee_credentials_downloader(urlstr, fullfilename)
            elif "404" in str(emsg):
                warnings.warn(f"{urlstr} cannot be found on server, skipped")
                fullfilename = warning_flag
            else:
                warnings.warn(f"Unknown error occurred - {emsg}")
                fullfilename = warning_flag

    return fullfilename def visit_spectra(
    dr=None,
    location=None,
    field=None,
    apogee=None,
    telescope=None,
    verbose=1,
    flag=None,
    commission=False,
):
    """
    Download the required individual spectra file a.k.a apStar or asStar

    :param dr: APOGEE DR
    :type dr: int
    :param location: Location ID [Optional]
    :type location: int
    :param field: Field [Optional]
    :type field: str
    :param apogee: Apogee ID
    :type apogee: str
    :param telescope: Telescope ID, for example 'apo25m' or 'lco25m'
    :type telescope: str
    :param verbose: verbose, set 0 to silent most logging
    :type verbose: int
    :param flag: 0: normal, 1: force to re-download
    :type flag: int
    :param commission: whether the spectra is taken during commissioning
    :type commission: bool

    :return: full file path and download in background if not found locally, False if cannot be found on server
    :rtype: str
    :History:
        | 2017-Nov-11 - Written - Henry Leung (University of Toronto)
        | 2018-Aug-31 - Updated - Henry Leung (University of Toronto)
    """
    dr = apogee_default_dr(dr=dr)

    # for DR16=<, location is expected to be none because field is used
    if (location is None and dr < 16) or (
        field is None and dr >= 16
    ):  # try to load info if not enough info
        global _ALLSTAR_TEMP
        if not str(f"dr{dr}") in _ALLSTAR_TEMP:
            _ALLSTAR_TEMP[f"dr{dr}"] = fits.getdata(allstar(dr=dr))
        if telescope is None:
            matched_idx = [
                np.nonzero(_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)[0]
            ][0]
        else:
            matched_idx = [
                np.nonzero(
                    [
                        (_ALLSTAR_TEMP[f"dr{dr}"]["APOGEE_ID"] == apogee)
                        & (_ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"] == telescope)
                    ]
                )
            ][0][1]
        if len(matched_idx) == 0:
            raise ValueError(
                f"No entry found in allstar DR{dr} met with your requirement!!"
            )

        location = (
            _ALLSTAR_TEMP[f"dr{dr}"]["LOCATION_ID"][matched_idx][0]
            if not location
            else location
        )
        field = (
            _ALLSTAR_TEMP[f"dr{dr}"]["FIELD"][matched_idx][0] if not field else field
        )
        telescope = (
            _ALLSTAR_TEMP[f"dr{dr}"]["TELESCOPE"][matched_idx][0]
            if not telescope
            else telescope
        )

    if dr == 13:
        reduce_prefix = "r6"
        str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/"
        if commission:
            filename = f"apStarC-{reduce_prefix}-{apogee}.fits"
        else:
            filename = f"apStar-{reduce_prefix}-{apogee}.fits"
        urlstr = str1 + filename
        hash_filename = f"{reduce_prefix}_stars_apo25m_{location}.sha1sum"

        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/",
            str(location),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

    elif dr == 14:
        reduce_prefix = "r8"
        str1 = f"https://data.sdss.org/sas/dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/{location}/"
        if commission:
            filename = f"apStarC-{reduce_prefix}-{apogee}.fits"
        else:
            filename = f"apStar-{reduce_prefix}-{apogee}.fits"
        urlstr = str1 + filename
        hash_filename = f"{reduce_prefix}_stars_apo25m_{location}.sha1sum"

        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/apo25m/",
            str(location),
        )
        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)

    elif dr == 16:
        reduce_prefix = "r12"
        str1 = f"https://data.sdss.org/sas/dr16/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/{field}/"
        if telescope == "lco25m":
            if commission:
                filename = f"asStarC-{reduce_prefix}-{apogee}.fits"
            else:
                filename = f"asStar-{reduce_prefix}-{apogee}.fits"
        else:
            if commission:
                filename = f"apStarC-{reduce_prefix}-{apogee}.fits"
            else:
                filename = f"apStar-{reduce_prefix}-{apogee}.fits"
        urlstr = str1 + filename
        hash_filename = f"{reduce_prefix}_stars_{telescope}_{field}.sha1sum"

        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/",
            str(f"{field}"),
        )

        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)
    elif dr == 17:
        reduce_prefix = "dr17"
        str1 = f"https://data.sdss.org/sas/dr17/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/{field}/"
        if telescope == "lco25m":
            if commission:
                filename = f"asStarC-{reduce_prefix}-{apogee}.fits"
            else:
                filename = f"asStar-{reduce_prefix}-{apogee}.fits"
        else:
            if commission:
                filename = f"apStarC-{reduce_prefix}-{apogee}.fits"
            else:
                filename = f"apStar-{reduce_prefix}-{apogee}.fits"
        urlstr = str1 + filename
        hash_filename = f"{reduce_prefix}_stars_{telescope}_{field}.sha1sum"

        fullfoldername = os.path.join(
            apogee_env(),
            f"dr{dr}/apogee/spectro/redux/{reduce_prefix}/stars/{telescope}/",
            str(f"{field}"),
        )

        if not os.path.exists(fullfoldername):
            os.makedirs(fullfoldername)
    else:
        raise ValueError("visit_spectra() only supports APOGEE DR13-DR17")

    # check hash file
    full_hash_filename = os.path.join(fullfoldername, hash_filename)
    if not os.path.isfile(full_hash_filename):
        # return warning flag if the location_id cannot even be found
        try:
            urllib.request.urlopen(str1)
        except urllib.error.HTTPError:
            return warning_flag
        urllib.request.urlretrieve(str1 + hash_filename, full_hash_filename)

    hash_list = np.loadtxt(full_hash_filename, dtype="str").T

    fullfilename = os.path.join(fullfoldername, filename)

    # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too
    # visit spectra has a different filename in checksum
    # handle the case where apogee_id cannot be found
    hash_idx = [
        i
        for i, item in enumerate(hash_list[1])
        if f"apStar-{reduce_prefix}-{apogee}" in item
    ]
    file_hash = hash_list[0][hash_idx]

    if os.path.isfile(fullfilename) and flag is None:
        checksum = filehash(fullfilename, algorithm="sha1")
        if checksum != file_hash and len(file_hash) != 0:
            warnings.warn(
                "File corruption detected, astroNN is attempting to download again"
            )
            visit_spectra(
                dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1
            )

        if verbose:
            logging.info(fullfilename + " was found!")

    elif not os.path.isfile(fullfilename) or flag == 1:
        try:
            urllib.request.urlretrieve(urlstr, fullfilename)
            logging.info(
                f"Downloaded DR{dr} individual visit file successfully to {fullfilename}"
            )
            checksum = filehash(fullfilename, algorithm="sha1")
            if checksum != file_hash and len(file_hash) != 0:
                warnings.warn(
                    "File corruption detected, astroNN is attempting to download again"
                )
                visit_spectra(
                    dr=dr, location=location, apogee=apogee, verbose=verbose, flag=1
                )
        except urllib.error.HTTPError as emsg:
            if "401" in str(emsg):
                fullfilename = __apogee_credentials_downloader(urlstr, fullfilename)
            elif "404" in str(emsg):
                warnings.warn(f"{urlstr} cannot be found on server, skipped")
                fullfilename = warning_flag
            else:
                warnings.warn(f"Unknown error occurred - {emsg}")
                fullfilename = warning_flag

    return fullfilename def apogeeid_digit(arr):
    """
    NAME:
        apogeeid_digit
    PURPOSE:
        Extract digits from apogeeid because its too painful to deal with APOGEE ID in h5py
    INPUT:
        arr (ndarray): apogee_id
    OUTPUT:
        apogee_id with digits only (ndarray)
    HISTORY:
        2017-Oct-26 - Written - Henry Leung (University of Toronto)
    """
    import numpy as np
    if isinstance(arr, np.ndarray) or isinstance(arr, list):
        arr_copy = np.array(arr)  # make a copy
        for i in range(arr_copy.shape[0]):
            arr_copy[i] = str(''.join(filter(str.isdigit, arr_copy[i])))
        return arr_copy
    else:
        return str(''.join(filter(str.isdigit, arr)))
import unittest
import numpy as np
import numpy.testing as npt
from astroNN.apogee import (
    gap_delete,
    apogee_default_dr,
    bitmask_decompositor,
    chips_split,
    bitmask_boolean,
    apogee_continuum,
    aspcap_mask,
    combined_spectra,
    visit_spectra,
)
from astroNN.apogee.apogee_shared import apogeeid_digit



class ApogeeToolsCase(unittest.TestCase):
    def test_apogee_tools(self):
        # Example data
        raw_spectra = np.ones((10, 8575))
        raw_spectrum = np.ones(8575)
        wrong_spectrum = np.ones(1024)

        gap_deleted = gap_delete(raw_spectra)
        self.assertEqual(gap_deleted.shape == (10, 7514), True)
        gap_deleted = gap_delete(raw_spectrum)
        self.assertEqual(gap_deleted.shape == (1, 7514), True)
        gap_deleted = gap_delete(raw_spectra, dr=12)
        self.assertEqual(gap_deleted.shape == (10, 7214), True)
        gap_deleted = gap_delete(raw_spectrum, dr=12)
        self.assertEqual(gap_deleted.shape == (1, 7214), True)
        self.assertRaises(EnvironmentError, gap_delete, wrong_spectrum)

        # check gaia default dr
        dr = apogee_default_dr()
        self.assertEqual(dr, 16)
        dr = apogee_default_dr(dr=3)
        self.assertEqual(dr, 3)

        # bitmask
        self.assertEqual(bitmask_decompositor(0), None)
        npt.assert_array_equal(bitmask_decompositor(1), [0])
        npt.assert_array_equal(bitmask_decompositor(3), [0, 1])
        npt.assert_array_equal(bitmask_boolean([0, 1, 2], [0]), [[False, True, False]])
        self.assertRaises(ValueError, bitmask_decompositor, -1)

        # chips_split
        blue, green, red = chips_split(raw_spectra)
        self.assertEqual(
            np.concatenate((blue, green, red), axis=1).shape == (10, 7514), True
        )
        blue, green, red = chips_split(raw_spectrum)
        self.assertEqual(
            np.concatenate((blue, green, red), axis=1).shape == (1, 7514), True
        )
        self.assertRaises(ValueError, chips_split, raw_spectra, dr=10)

    def test_apogee_continuum(self):
        raw_spectra = np.ones((10, 8575)) * 2
        raw_spectra_err = np.zeros((10, 8575))
        # continuum
        cont_spectra, cont_spectra_arr = apogee_continuum(raw_spectra, raw_spectra_err)
        self.assertAlmostEqual(float(np.mean(cont_spectra)), 1.0)

    def test_apogee_digit_extractor(self):
        # Test apogeeid digit extractor
        # just to make no error
        apogeeid_digit(["2M00380508+5608579", "2M00380508+5608579"])
        apogeeid_digit(np.array(["2M00380508+5608579", "2M00380508+5608579"]))

        # check accuracy
        self.assertEqual(apogeeid_digit("2M00380508+5608579"), "2003805085608579")
        npt.assert_array_equal(
            apogeeid_digit(np.array(["2M00380508+5608579", "2M00380508+5608579"])),
            ["2003805085608579", "2003805085608579"],
        )

    def test_aspcap_mask(self):
        self.assertEqual(np.all(aspcap_mask("C1") == aspcap_mask("ci")), True)
        self.assertEqual(np.all(aspcap_mask("TIII") == aspcap_mask("ti2")), True)
        # assert for example dr=1 is not supported
        self.assertRaises(ValueError, aspcap_mask, "al", 1)
        # Make sure if element not found, the case is nicely handled
        self.assertEqual(aspcap_mask("abc"), None)


class ApogeeDownloaderCase(unittest.TestCase):
    def test_apogee_combined_download(self):
        """
        Test APOGEE combined spectra downloading function, assert functions can deal with missing files
        """
        # make sure the download works correctly
        combined_spectra(dr=13, location=4405, apogee="2M19060637+4717296")
        combined_spectra(dr=14, location=4405, apogee="2M19060637+4717296")
        combined_spectra(
            dr=16, field="K06_078+16", telescope="apo25m", apogee="2M19060637+4717296"
        )
        combined_spectra(
            dr=17, field="K06_078+16", telescope="apo25m", apogee="2M19060637+4717296"
        )
        # assert False is returning if file not found
        self.assertEqual(
            combined_spectra(dr=13, location=4406, apogee="2M19060637+4717296"), False
        )
        self.assertEqual(
            combined_spectra(dr=14, location=4406, apogee="2M19060637+4717296"), False
        )
        self.assertEqual(
            combined_spectra(
                dr=16,
                field="K06_078+17",
                telescope="apo25m",
                apogee="2M19060637+4717296",
            ),
            False,
        )
        self.assertEqual(

            combined_spectra(