{"action":"created","comment":{"url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/comments/233545481","pull_request_review_id":174985415,"id":233545481,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzMzU0NTQ4MQ==","diff_hunk":"@@ -0,0 +1,737 @@\n+# See COPYRIGHT file at the top of the source tree.\n+#\n+# This file is part of fgcmcal.\n+#\n+# Developed for the LSST Data Management System.\n+# This product includes software developed by the LSST Project\n+# (https://www.lsst.org).\n+# See the COPYRIGHT file at the top-level directory of this distribution\n+# for details of code ownership.\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the GNU General Public License\n+# along with this program.  If not, see <https://www.gnu.org/licenses/>.\n+\n+import sys\n+import traceback\n+import copy\n+\n+import numpy as np\n+import healpy as hp\n+import esutil\n+\n+import lsst.pex.config as pexConfig\n+import lsst.pipe.base as pipeBase\n+from lsst.afw.image import TransmissionCurve\n+from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask\n+from lsst.pipe.tasks.photoCal import PhotoCalTask\n+from .fgcmFitCycle import FgcmFitCycleTask\n+import lsst.geom\n+import lsst.afw.image as afwImage\n+import lsst.afw.math as afwMath\n+import lsst.afw.table as afwTable\n+from lsst.meas.algorithms import IndexerRegistry\n+from lsst.meas.algorithms import DatasetConfig\n+\n+import fgcm\n+\n+__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner']\n+\n+\n+class FgcmOutputProductsConfig(pexConfig.Config):\n+    \"\"\"Config for FgcmOutputProductsTask\"\"\"\n+\n+    cycleNumber = pexConfig.Field(\n+        doc=\"Final fit cycle from FGCM fit\",\n+        dtype=int,\n+        default=None,\n+    )\n+\n+    # The following fields refer to calibrating from a reference\n+    # catalog, but in the future this might need to be expanded\n+    doReferenceCalibration = pexConfig.Field(\n+        doc=\"Apply 'absolute' calibration from reference catalog\",\n+        dtype=bool,\n+        default=True,\n+    )\n+    doRefcatOutput = pexConfig.Field(\n+        doc=\"Output standard stars in reference catalog format\",\n+        dtype=bool,\n+        default=True,\n+    )\n+    doAtmosphereOutput = pexConfig.Field(\n+        doc=\"Output atmospheres in transmission format\",\n+        dtype=bool,\n+        default=True,\n+    )\n+    doZeropointOutput = pexConfig.Field(\n+        doc=\"Output zeropoints in jointcal_photoCalib format\",\n+        dtype=bool,\n+        default=True,\n+    )\n+    refObjLoader = pexConfig.ConfigurableField(\n+        target=LoadIndexedReferenceObjectsTask,\n+        doc=\"reference object loader for 'absolute' photometric calibration\",\n+    )\n+    photoCal = pexConfig.ConfigurableField(\n+        target=PhotoCalTask,\n+        doc=\"task to perform 'absolute' calibration\",\n+    )\n+    referencePixelizationNside = pexConfig.Field(\n+        doc=\"Healpix nside to pixelize catalog to compare to reference\",\n+        dtype=int,\n+        default=64,\n+    )\n+    referencePixelizationMinStars = pexConfig.Field(\n+        doc=\"Minimum number of stars per pixel to select for comparison\",\n+        dtype=int,\n+        default=200,\n+    )\n+    referenceMinMatch = pexConfig.Field(\n+        doc=\"Minimum number of stars matched to reference catalog to be used in statistics\",\n+        dtype=int,\n+        default=50,\n+    )\n+    referencePixelizationNPixels = pexConfig.Field(\n+        doc=\"Number of pixels to sample to do comparison\",\n+        dtype=int,\n+        default=100,\n+    )\n+    datasetConfig = pexConfig.ConfigField(\n+        dtype=DatasetConfig,\n+        doc=\"Configuration for writing/reading ingested catalog\",\n+    )\n+\n+    def setDefaults(self):\n+        pexConfig.Config.setDefaults(self)\n+\n+        self.photoCal.applyColorTerms = True\n+        self.photoCal.fluxField = 'flux'\n+        self.photoCal.magErrFloor = 0.003\n+        self.photoCal.match.referenceSelection.doSignalToNoise = True\n+        self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0\n+        self.photoCal.match.sourceSelection.doSignalToNoise = True\n+        self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0\n+        self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'flux'\n+        self.photoCal.match.sourceSelection.signalToNoise.errField = 'fluxErr'\n+        self.photoCal.match.sourceSelection.doFlags = True\n+        self.photoCal.match.sourceSelection.flags.good = []\n+        self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar']\n+        self.photoCal.match.sourceSelection.doUnresolved = False\n+        self.datasetConfig.ref_dataset_name = 'fgcm_stars'\n+\n+\n+class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner):\n+    \"\"\"Subclass of TaskRunner for fgcmOutputProductsTask\n+\n+    fgcmOutputProductsTask.run() takes one argument, the butler, and\n+    does not run on any data in the repository.\n+    This runner does not use any parallelization.\n+    \"\"\"\n+\n+    @staticmethod\n+    def getTargetList(parsedCmd):\n+        \"\"\"\n+        Return a list with one element, the butler.\n+        \"\"\"\n+        return [parsedCmd.butler]\n+\n+    def precall(self, parsedCmd):\n+        return True\n+\n+    def __call__(self, butler):\n+        \"\"\"\n+        Parameters\n+        ----------\n+        butler: lsst.daf.persistence.Butler\n+\n+        Returns\n+        -------\n+        None if self.doReturnResults is False\n+        An empty list if self.doReturnResults is True\n+        \"\"\"\n+        task = self.TaskClass(butler=butler, config=self.config, log=self.log)\n+\n+        exitStatus = 0\n+        if self.doRaise:\n+            results = task.runDataRef(butler)\n+        else:\n+            try:\n+                results = task.runDataRef(butler)\n+            except Exception as e:\n+                exitStatus = 1\n+                task.log.fatal(\"Failed: %s\" % e)\n+                if not isinstance(e, pipeBase.TaskError):\n+                    traceback.print_exc(file=sys.stderr)\n+\n+        task.writeMetadata(butler)\n+\n+        if self.doReturnResults:\n+            # Not much in terms of results to return (empty list)\n+            return [pipeBase.Struct(exitStatus=exitStatus,\n+                                    results=results)]\n+        else:\n+            return [pipeBase.Struct(exitStatus=exitStatus)]\n+\n+    # turn off any multiprocessing\n+\n+    def run(self, parsedCmd):\n+        \"\"\"\n+        Run the task, with no multiprocessing\n+\n+        Parameters\n+        ----------\n+        parsedCmd: ArgumentParser parsed command line\n+        \"\"\"\n+\n+        resultList = []\n+\n+        if self.precall(parsedCmd):\n+            # profileName = parsedCmd.profile if hasattr(parsedCmd, \"profile\") else None\n+            # log = parsedCmd.log\n+            targetList = self.getTargetList(parsedCmd)\n+            # make sure that we only get 1\n+            resultList = self(targetList[0])\n+\n+        return resultList\n+\n+\n+class FgcmOutputProductsTask(pipeBase.CmdLineTask):\n+    \"\"\"\n+    Output products from FGCM global calibration.\n+\n+    Currently this is atmosphere transmission tables, in the future will\n+    include zeropoint files.\n+    \"\"\"\n+\n+    ConfigClass = FgcmOutputProductsConfig\n+    RunnerClass = FgcmOutputProductsRunner\n+    _DefaultName = \"fgcmOutputProducts\"\n+\n+    def __init__(self, butler=None, **kwargs):\n+        \"\"\"\n+        Instantiate an fgcmOutputProductsTask.\n+\n+        Parameters\n+        ----------\n+        butler : lsst.daf.persistence.Butler\n+        \"\"\"\n+\n+        pipeBase.CmdLineTask.__init__(self, **kwargs)\n+\n+        if self.config.doReferenceCalibration:\n+            # We need the ref obj loader to get the flux field\n+            self.makeSubtask(\"refObjLoader\", butler=butler)\n+\n+        if self.config.doRefcatOutput:\n+            self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name](\n+                self.config.datasetConfig.indexer.active)\n+\n+    @classmethod\n+    def _makeArgumentParser(cls):\n+        \"\"\"Create an argument parser\"\"\"\n+\n+        parser = pipeBase.ArgumentParser(name=cls._DefaultName)\n+\n+        return parser\n+\n+    # no saving of metadata for now\n+    def _getMetadataName(self):\n+        return None\n+\n+    @pipeBase.timeMethod\n+    def runDataRef(self, butler):\n+        \"\"\"\n+        Make FGCM output products for use in the stack\n+\n+        Parameters\n+        ----------\n+        butler:  lsst.daf.persistence.Butler\n+\n+        Returns\n+        -------\n+        Empty list\n+        \"\"\"\n+\n+        # Check to make sure that the fgcmBuildStars config exists, to retrieve\n+        # the visit and ccd dataset tags\n+        if not butler.datasetExists('fgcmBuildStars_config'):\n+            raise RuntimeError(\"Cannot find fgcmBuildStars_config, which is prereq for fgcmOutputProducts\")\n+\n+        fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config')\n+        self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName\n+        self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName\n+        self.filterToBand = fgcmBuildStarsConfig.filterToBand\n+\n+        # Make sure that the fit config exists, to retrieve bands and other info\n+        if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber):\n+            raise RuntimeError(\"Cannot find fgcmFitCycle_config from cycle %d \" % (self.config.cycleNumber) +\n+                               \"which is required for fgcmOutputProducts.\")\n+\n+        fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)\n+        self.bands = fitCycleConfig.bands\n+        self.superStarSubCcd = fitCycleConfig.superStarSubCcd\n+        self.chebyshevOrder = fitCycleConfig.superStarSubCcdChebyshevOrder\n+\n+        # And make sure that the atmosphere was output properly\n+        if not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber):\n+            raise RuntimeError(\"The specified fgcm cycle (%d) has not been run on this repo.\" %\n+                               (self.config.cycleNumber))\n+\n+        # And make sure this is the last cycle\n+        if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1):\n+            raise RuntimeError(\"The task fgcmOutputProducts should only be run\"\n+                               \"on the final fit cycle products\")\n+\n+        # Check if we need to run one final cycle to generate standards\n+        if not butler.datasetExists('fgcmStandardStars', fgcmcycle=self.config.cycleNumber):\n+            self.log.info(\"Standard stars will be generated using final fgcmFitCycle settings\")\n+            self._generateFgcmStandardStars(butler)\n+            # Increment the cycle number\n+            self.useCycle = self.config.cycleNumber + 1\n+        else:\n+            self.useCycle = self.config.cycleNumber\n+\n+        # And make sure that the atmosphere was output properly\n+        if not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.useCycle):\n+            raise RuntimeError(\"The latest fgcm cycle (%d) has not been run on this repo.\" %\n+                               (self.useCycle))\n+\n+        if self.config.doReferenceCalibration:\n+            offsets = self._computeReferenceOffsets(butler)\n+        else:\n+            offsets = np.zeros(len(self.bands))\n+\n+        # Output the standard stars in stack format\n+        if self.config.doRefcatOutput:\n+            self._outputStandardStars(butler, offsets)\n+\n+        # Output the gray zeropoints\n+        if self.config.doZeropointOutput:\n+            self._outputZeropoints(butler, offsets)\n+\n+        # Output the atmospheres\n+        if self.config.doAtmosphereOutput:\n+            self._outputAtmospheres(butler)\n+\n+        # We return the zp offsets\n+        return pipeBase.Struct(offsets=offsets)\n+\n+    def _generateFgcmStandardStars(self, butler):\n+        \"\"\"\n+        Call fgcmFitCycle with settings to output the standard stars.\n+\n+        Parameters\n+        ----------\n+        butler: lsst.daf.persistence.Butler\n+        \"\"\"\n+\n+        fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber)\n+        fitCycleConfig.cycleNumber += 1\n+        fitCycleConfig.maxIter = 0\n+        fitCycleConfig.outputStandards = True\n+\n+        fitCycleTask = FgcmFitCycleTask(butler=butler, config=fitCycleConfig)\n+        fitCycleTask.runDataRef(butler)\n+\n+    def _computeReferenceOffsets(self, butler):\n+        \"\"\"\n+        Compute offsets relative to a reference catalog.\n+\n+        Parameters\n+        ----------\n+        butler: lsst.daf.persistence.Butler\n+\n+        Returns\n+        -------\n+        offsets: np.array of floats\n+           Per band zeropoint offsets\n+        \"\"\"\n+\n+        # Load the stars\n+        stars = butler.get('fgcmStandardStars', fgcmcycle=self.useCycle)\n+\n+        # Only use stars that are observed in all the bands\n+        minObs = stars['ngood'].min(axis=1)\n+\n+        # Depending on how things work, this might need to be configurable\n+        # However, I think that best results will occur when we use the same\n+        # pixels to do the calibration for all the bands, so I think this\n+        # is the right choice.\n+\n+        goodStars = (minObs >= 1)\n+        stars = stars[goodStars]\n+\n+        self.log.info(\"Found %d stars with at least 1 good observation in each band\" %\n+                      (len(stars)))\n+\n+        # We have to make a table for each pixel with flux/fluxErr\n+        sourceMapper = afwTable.SchemaMapper(stars.schema)\n+        sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())\n+        sourceMapper.editOutputSchema().addField('flux', type=np.float64, doc=\"flux\")\n+        sourceMapper.editOutputSchema().addField('fluxErr', type=np.float64, doc=\"flux error\")\n+        badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',\n+                                                              type='Flag', doc=\"bad flag\")\n+\n+        # The exposure is used to record the filter name\n+        exposure = afwImage.ExposureF()\n+\n+        # Split up the stars (units are radians)\n+        theta = np.pi / 2. - stars['coord_dec']\n+        phi = stars['coord_ra']\n+\n+        ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)\n+        h, rev = esutil.stat.histogram(ipring, rev=True)\n+\n+        gdpix, = np.where(h >= self.config.referencePixelizationMinStars)\n+\n+        self.log.info(\"Found %d pixels (nside=%d) with at least %d good stars\" %\n+                      (gdpix.size,\n+                       self.config.referencePixelizationNside,\n+                       self.config.referencePixelizationMinStars))\n+\n+        if gdpix.size < self.config.referencePixelizationNPixels:\n+            self.log.warn(\"Found fewer good pixels (%d) than preferred in configuration (%d)\" %\n+                          (gdpix.size, self.config.referencePixelizationNPixels))\n+        else:\n+            # Sample out the pixels we want to use\n+            gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False)\n+\n+        results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'),\n+                                              ('nstar', 'i4', len(self.bands)),\n+                                              ('nmatch', 'i4', len(self.bands)),\n+                                              ('zp', 'f4', len(self.bands)),\n+                                              ('zpErr', 'f4', len(self.bands))])\n+        results['hpix'] = ipring[rev[rev[gdpix]]]\n+\n+        # We need a boolean index to deal with catalogs...\n+        selected = np.zeros(len(stars), dtype=np.bool)\n+\n+        refFluxFields = [None] * len(self.bands)\n+\n+        for p, pix in enumerate(gdpix):\n+            i1a = rev[rev[pix]: rev[pix + 1]]\n+\n+            # Need to convert to a boolean array\n+            selected[:] = False\n+            selected[i1a] = True\n+\n+            for b, band in enumerate(self.bands):\n+\n+                sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())\n+                sourceCat.reserve(len(i1a))\n+                sourceCat.extend(stars[selected], mapper=sourceMapper)\n+                sourceCat['flux'] = afwImage.fluxFromABMag(stars['mag_std_noabs'][selected, b])\n+                sourceCat['fluxErr'] = afwImage.fluxErrFromABMagErr(stars['magErr_std'][selected, b],\n+                                                                    stars['mag_std_noabs'][selected, b])\n+\n+                # Make sure we only use stars that have valid measurements\n+                # (This is perhaps redundant with requirements above that the\n+                # stars be observed in all bands, but it can't hurt)\n+                badStar = (stars['mag_std_noabs'][selected, b] > 90.0)\n+                for rec in sourceCat[badStar]:\n+                    rec.set(badStarKey, True)\n+\n+                exposure.setFilter(afwImage.Filter(band))\n+\n+                if refFluxFields[b] is None:\n+                    # Need to find the flux field in the reference catalog\n+                    # to work around limitations of DirectMatch in PhotoCal\n+                    ctr = stars[0].getCoord()\n+                    rad = 0.05 * lsst.geom.degrees\n+                    refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band)\n+                    refFluxFields[b] = refDataTest.fluxField\n+\n+                # Make a copy of the config so that we can modify it\n+                calConfig = copy.copy(self.config.photoCal.value)\n+                calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b]\n+                calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err'\n+                calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,\n+                                                      config=calConfig,\n+                                                      schema=sourceCat.getSchema())\n+\n+                struct = calTask.run(exposure, sourceCat)\n+\n+                results['nstar'][p, b] = len(i1a)\n+                results['nmatch'][p, b] = len(struct.arrays.refMag)\n+                results['zp'][p, b] = struct.zp\n+                results['zpErr'][p, b] = struct.sigma\n+\n+        # And compute the summary statistics\n+        offsets = np.zeros(len(self.bands))\n+\n+        for b, band in enumerate(self.bands):\n+            # make configurable\n+            ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch)\n+            offsets[b] = np.median(results['zp'][ok, b])\n+            madSigma = 1.4826 * np.median(np.abs(results['zp'][ok, b] - offsets[b]))\n+            self.log.info(\"Reference catalog offset for %s band: %.6f +/- %.6f\" %\n+                          (band, offsets[b], madSigma))\n+\n+        return offsets\n+\n+    def _outputStandardStars(self, butler, offsets):\n+        \"\"\"\n+        Output standard stars in indexed reference catalog format.\n+\n+        Parameters\n+        ----------\n+        butler: lsst.daf.persistence.Butler\n+        offsets: np.array of floats\n+           Per band zeropoint offsets\n+        \"\"\"\n+\n+        self.log.info(\"Outputting standard stars to %s\" % (self.config.datasetConfig.ref_dataset_name))\n+\n+        # Load the stars (this is the full set of stars, no cuts)\n+        stars = butler.get('fgcmStandardStars', fgcmcycle=self.useCycle)\n+\n+        # We assume these are returned in radians, because the units interface\n+        # required a loop over a giant catalog.\n+        # TODO: figure out check of native units\n+        indices = np.array(self.indexer.indexPoints(np.degrees(stars['coord_ra']),\n+                                                    np.degrees(stars['coord_dec'])))\n+\n+        formattedCat = self._formatCatalog(stars, offsets)\n+\n+        # Write the master schema\n+        dataId = self.indexer.makeDataId('master_schema',\n+                                         self.config.datasetConfig.ref_dataset_name)\n+        butler.put(afwTable.SimpleCatalog(formattedCat.schema), 'ref_cat', dataId=dataId)\n+\n+        # Break up the pixels using a histogram\n+        h, rev = esutil.stat.histogram(indices, rev=True)\n+        gd, = np.where(h > 0)\n+        selected = np.zeros(len(formattedCat), dtype=np.bool)\n+        for i in gd:\n+            i1a = rev[rev[i]: rev[i + 1]]\n+\n+            # Turn into a boolean array\n+            selected[:] = False\n+            selected[i1a] = True\n+\n+            # Write the individual pixel\n+            dataId = self.indexer.makeDataId(indices[i1a[0]],\n+                                             self.config.datasetConfig.ref_dataset_name)\n+            butler.put(formattedCat[selected], 'ref_cat', dataId=dataId)\n+\n+        # And save the dataset configuration\n+        dataId = self.indexer.makeDataId(None, self.config.datasetConfig.ref_dataset_name)\n+        butler.put(self.config.datasetConfig, 'ref_cat_config', dataId=dataId)\n+\n+        self.log.info(\"Done outputting standard stars.\")\n+\n+    def _formatCatalog(self, fgcmStarCat, offsets):\n+        \"\"\"\n+        Turn an FGCM-formatted star catalog, applying zp offsets.\n+\n+        parameters\n+        ----------\n+        fgcmStarCat: SimpleCatalog\n+           SimpleCatalog as output by fgcmcal\n+        offsets: list with len(self.bands) entries\n+           Zeropoint offsets to apply\n+\n+        returns\n+        -------\n+        formattedCat: SimpleCatalog\n+           SimpleCatalog suitable for ref_cat\n+        \"\"\"\n+\n+        sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)\n+        sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())\n+        for band in self.bands:\n+            sourceMapper.editOutputSchema().addField('%s_flux' % (band), type=np.float64)\n+            sourceMapper.editOutputSchema().addField('%s_fluxErr' % (band), type=np.float64)\n+            sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.float64)\n+\n+        formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())\n+        formattedCat.reserve(len(fgcmStarCat))\n+        formattedCat.extend(fgcmStarCat, mapper=sourceMapper)\n+\n+        for b, band in enumerate(self.bands):\n+            mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]\n+            flux = afwImage.fluxFromABMag(mag)\n+            fluxErr = afwImage.fluxErrFromABMagErr(fgcmStarCat['magErr_std'][:, b], mag)\n+            formattedCat['%s_flux' % (band)][:] = flux\n+            formattedCat['%s_fluxErr' % (band)][:] = fluxErr\n+            formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]\n+\n+        return formattedCat\n+\n+    def _outputZeropoints(self, butler, offsets):\n+        \"\"\"\n+        Output the zeropoints in jointcal_photoCalib format.\n+\n+        Parameters\n+        ----------\n+        butler: lsst.daf.persistence.Butler\n+        \"\"\"\n+\n+        self.log.info(\"Outputting jointcal_photoCalib objects\")\n+\n+        zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.useCycle)\n+        visitCat = butler.get('fgcmVisitCatalog')\n+\n+        # Only output those that we have a calibration\n+        selected = (zptCat['fgcmFlag'] < 16)\n+\n+        # Get the mapping from filtername to dataId filter name\n+        filterMapping = {}\n+        nFound = 0\n+        for rec in zptCat[selected]:\n+            if rec['filtername'] in filterMapping:\n+                continue\n+            dataId = {self.visitDataRefName: int(rec['visit']),\n+                      self.ccdDataRefName: int(rec['ccd'])}\n+            dataRef = butler.dataRef('raw', dataId=dataId)\n+            filterMapping[rec['filtername']] = dataRef.dataId['filter']\n+            nFound += 1\n+            if nFound == len(self.filterToBand):\n+                break\n+\n+        # Get a mapping from filtername to the offsets\n+        offsetMapping = {}\n+        for f in self.filterToBand:\n+            offsetMapping[f] = offsets[self.bands.index(self.filterToBand[f])]\n+\n+        # Get a mapping from \"ccd\" to the ccd index used for the scaling\n+        camera = butler.get('camera')\n+        ccdMapping = {}\n+        for ccdIndex, detector in enumerate(camera):\n+            ccdMapping[detector.getId()] = ccdIndex\n+\n+        # And a mapping to get the flat-field scaling values\n+        scalingMapping = {}\n+        for rec in visitCat:\n+            scalingMapping[rec['visit']] = rec['scaling']\n+\n+        # Make a fixed variable to hold the parameters to build a ChebyshevBoundedField\n+        orderPlus1 = self.chebyshevOrder + 1\n+\n+        pars = np.zeros((orderPlus1, orderPlus1))\n+\n+        for rec in zptCat[selected]:\n+\n+            if self.superStarSubCcd:\n+                # Spatially varying zeropoint","path":"python/lsst/fgcmcal/fgcmOutputProducts.py","position":626,"original_position":626,"commit_id":"49c6ef5ccbb9ecaf5267580af125f1144a61f227","original_commit_id":"49c6ef5ccbb9ecaf5267580af125f1144a61f227","user":{"login":"erykoff","id":5167891,"node_id":"MDQ6VXNlcjUxNjc4OTE=","avatar_url":"https://avatars3.githubusercontent.com/u/5167891?v=4","gravatar_id":"","url":"https://api.github.com/users/erykoff","html_url":"https://github.com/erykoff","followers_url":"https://api.github.com/users/erykoff/followers","following_url":"https://api.github.com/users/erykoff/following{/other_user}","gists_url":"https://api.github.com/users/erykoff/gists{/gist_id}","starred_url":"https://api.github.com/users/erykoff/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erykoff/subscriptions","organizations_url":"https://api.github.com/users/erykoff/orgs","repos_url":"https://api.github.com/users/erykoff/repos","events_url":"https://api.github.com/users/erykoff/events{/privacy}","received_events_url":"https://api.github.com/users/erykoff/received_events","type":"User","site_admin":false},"body":"I have made this change, but we need to discuss the granularity of tests.","created_at":"2018-11-14T17:29:56Z","updated_at":"2018-11-14T17:29:56Z","html_url":"https://github.com/parejkoj/fgcmcal/pull/1#discussion_r233545481","pull_request_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/comments/233545481"},"html":{"href":"https://github.com/parejkoj/fgcmcal/pull/1#discussion_r233545481"},"pull_request":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1"}},"in_reply_to_id":233303780},"pull_request":{"url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1","id":229202348,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI5MjAyMzQ4","html_url":"https://github.com/parejkoj/fgcmcal/pull/1","diff_url":"https://github.com/parejkoj/fgcmcal/pull/1.diff","patch_url":"https://github.com/parejkoj/fgcmcal/pull/1.patch","issue_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/1","number":1,"state":"open","locked":false,"title":"Flattened all commits, so I can try making a giant PR.","user":{"login":"parejkoj","id":1606089,"node_id":"MDQ6VXNlcjE2MDYwODk=","avatar_url":"https://avatars3.githubusercontent.com/u/1606089?v=4","gravatar_id":"","url":"https://api.github.com/users/parejkoj","html_url":"https://github.com/parejkoj","followers_url":"https://api.github.com/users/parejkoj/followers","following_url":"https://api.github.com/users/parejkoj/following{/other_user}","gists_url":"https://api.github.com/users/parejkoj/gists{/gist_id}","starred_url":"https://api.github.com/users/parejkoj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/parejkoj/subscriptions","organizations_url":"https://api.github.com/users/parejkoj/orgs","repos_url":"https://api.github.com/users/parejkoj/repos","events_url":"https://api.github.com/users/parejkoj/events{/privacy}","received_events_url":"https://api.github.com/users/parejkoj/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-11-07T23:26:10Z","updated_at":"2018-11-14T17:29:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7b35de1b24571c7f879114195fa2c1ab7ec13c8e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1/commits","review_comments_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1/comments","review_comment_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/comments{/number}","comments_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/1/comments","statuses_url":"https://api.github.com/repos/parejkoj/fgcmcal/statuses/49c6ef5ccbb9ecaf5267580af125f1144a61f227","head":{"label":"parejkoj:u/parejkoj/review-flattened","ref":"u/parejkoj/review-flattened","sha":"49c6ef5ccbb9ecaf5267580af125f1144a61f227","user":{"login":"parejkoj","id":1606089,"node_id":"MDQ6VXNlcjE2MDYwODk=","avatar_url":"https://avatars3.githubusercontent.com/u/1606089?v=4","gravatar_id":"","url":"https://api.github.com/users/parejkoj","html_url":"https://github.com/parejkoj","followers_url":"https://api.github.com/users/parejkoj/followers","following_url":"https://api.github.com/users/parejkoj/following{/other_user}","gists_url":"https://api.github.com/users/parejkoj/gists{/gist_id}","starred_url":"https://api.github.com/users/parejkoj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/parejkoj/subscriptions","organizations_url":"https://api.github.com/users/parejkoj/orgs","repos_url":"https://api.github.com/users/parejkoj/repos","events_url":"https://api.github.com/users/parejkoj/events{/privacy}","received_events_url":"https://api.github.com/users/parejkoj/received_events","type":"User","site_admin":false},"repo":{"id":156620952,"node_id":"MDEwOlJlcG9zaXRvcnkxNTY2MjA5NTI=","name":"fgcmcal","full_name":"parejkoj/fgcmcal","private":false,"owner":{"login":"parejkoj","id":1606089,"node_id":"MDQ6VXNlcjE2MDYwODk=","avatar_url":"https://avatars3.githubusercontent.com/u/1606089?v=4","gravatar_id":"","url":"https://api.github.com/users/parejkoj","html_url":"https://github.com/parejkoj","followers_url":"https://api.github.com/users/parejkoj/followers","following_url":"https://api.github.com/users/parejkoj/following{/other_user}","gists_url":"https://api.github.com/users/parejkoj/gists{/gist_id}","starred_url":"https://api.github.com/users/parejkoj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/parejkoj/subscriptions","organizations_url":"https://api.github.com/users/parejkoj/orgs","repos_url":"https://api.github.com/users/parejkoj/repos","events_url":"https://api.github.com/users/parejkoj/events{/privacy}","received_events_url":"https://api.github.com/users/parejkoj/received_events","type":"User","site_admin":false},"html_url":"https://github.com/parejkoj/fgcmcal","description":"Forward Global Calibration Method","fork":true,"url":"https://api.github.com/repos/parejkoj/fgcmcal","forks_url":"https://api.github.com/repos/parejkoj/fgcmcal/forks","keys_url":"https://api.github.com/repos/parejkoj/fgcmcal/keys{/key_id}","collaborators_url":"https://api.github.com/repos/parejkoj/fgcmcal/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/parejkoj/fgcmcal/teams","hooks_url":"https://api.github.com/repos/parejkoj/fgcmcal/hooks","issue_events_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/events{/number}","events_url":"https://api.github.com/repos/parejkoj/fgcmcal/events","assignees_url":"https://api.github.com/repos/parejkoj/fgcmcal/assignees{/user}","branches_url":"https://api.github.com/repos/parejkoj/fgcmcal/branches{/branch}","tags_url":"https://api.github.com/repos/parejkoj/fgcmcal/tags","blobs_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/refs{/sha}","trees_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/trees{/sha}","statuses_url":"https://api.github.com/repos/parejkoj/fgcmcal/statuses/{sha}","languages_url":"https://api.github.com/repos/parejkoj/fgcmcal/languages","stargazers_url":"https://api.github.com/repos/parejkoj/fgcmcal/stargazers","contributors_url":"https://api.github.com/repos/parejkoj/fgcmcal/contributors","subscribers_url":"https://api.github.com/repos/parejkoj/fgcmcal/subscribers","subscription_url":"https://api.github.com/repos/parejkoj/fgcmcal/subscription","commits_url":"https://api.github.com/repos/parejkoj/fgcmcal/commits{/sha}","git_commits_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/commits{/sha}","comments_url":"https://api.github.com/repos/parejkoj/fgcmcal/comments{/number}","issue_comment_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/comments{/number}","contents_url":"https://api.github.com/repos/parejkoj/fgcmcal/contents/{+path}","compare_url":"https://api.github.com/repos/parejkoj/fgcmcal/compare/{base}...{head}","merges_url":"https://api.github.com/repos/parejkoj/fgcmcal/merges","archive_url":"https://api.github.com/repos/parejkoj/fgcmcal/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/parejkoj/fgcmcal/downloads","issues_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues{/number}","pulls_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls{/number}","milestones_url":"https://api.github.com/repos/parejkoj/fgcmcal/milestones{/number}","notifications_url":"https://api.github.com/repos/parejkoj/fgcmcal/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/parejkoj/fgcmcal/labels{/name}","releases_url":"https://api.github.com/repos/parejkoj/fgcmcal/releases{/id}","deployments_url":"https://api.github.com/repos/parejkoj/fgcmcal/deployments","created_at":"2018-11-07T23:15:22Z","updated_at":"2018-11-07T23:15:24Z","pushed_at":"2018-11-14T01:46:48Z","git_url":"git://github.com/parejkoj/fgcmcal.git","ssh_url":"git@github.com:parejkoj/fgcmcal.git","clone_url":"https://github.com/parejkoj/fgcmcal.git","svn_url":"https://github.com/parejkoj/fgcmcal","homepage":"","size":691,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"parejkoj:first-commit","ref":"first-commit","sha":"41c06e5b4b35bc606cb88b0fbe1f8f43fe60e12c","user":{"login":"parejkoj","id":1606089,"node_id":"MDQ6VXNlcjE2MDYwODk=","avatar_url":"https://avatars3.githubusercontent.com/u/1606089?v=4","gravatar_id":"","url":"https://api.github.com/users/parejkoj","html_url":"https://github.com/parejkoj","followers_url":"https://api.github.com/users/parejkoj/followers","following_url":"https://api.github.com/users/parejkoj/following{/other_user}","gists_url":"https://api.github.com/users/parejkoj/gists{/gist_id}","starred_url":"https://api.github.com/users/parejkoj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/parejkoj/subscriptions","organizations_url":"https://api.github.com/users/parejkoj/orgs","repos_url":"https://api.github.com/users/parejkoj/repos","events_url":"https://api.github.com/users/parejkoj/events{/privacy}","received_events_url":"https://api.github.com/users/parejkoj/received_events","type":"User","site_admin":false},"repo":{"id":156620952,"node_id":"MDEwOlJlcG9zaXRvcnkxNTY2MjA5NTI=","name":"fgcmcal","full_name":"parejkoj/fgcmcal","private":false,"owner":{"login":"parejkoj","id":1606089,"node_id":"MDQ6VXNlcjE2MDYwODk=","avatar_url":"https://avatars3.githubusercontent.com/u/1606089?v=4","gravatar_id":"","url":"https://api.github.com/users/parejkoj","html_url":"https://github.com/parejkoj","followers_url":"https://api.github.com/users/parejkoj/followers","following_url":"https://api.github.com/users/parejkoj/following{/other_user}","gists_url":"https://api.github.com/users/parejkoj/gists{/gist_id}","starred_url":"https://api.github.com/users/parejkoj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/parejkoj/subscriptions","organizations_url":"https://api.github.com/users/parejkoj/orgs","repos_url":"https://api.github.com/users/parejkoj/repos","events_url":"https://api.github.com/users/parejkoj/events{/privacy}","received_events_url":"https://api.github.com/users/parejkoj/received_events","type":"User","site_admin":false},"html_url":"https://github.com/parejkoj/fgcmcal","description":"Forward Global Calibration Method","fork":true,"url":"https://api.github.com/repos/parejkoj/fgcmcal","forks_url":"https://api.github.com/repos/parejkoj/fgcmcal/forks","keys_url":"https://api.github.com/repos/parejkoj/fgcmcal/keys{/key_id}","collaborators_url":"https://api.github.com/repos/parejkoj/fgcmcal/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/parejkoj/fgcmcal/teams","hooks_url":"https://api.github.com/repos/parejkoj/fgcmcal/hooks","issue_events_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/events{/number}","events_url":"https://api.github.com/repos/parejkoj/fgcmcal/events","assignees_url":"https://api.github.com/repos/parejkoj/fgcmcal/assignees{/user}","branches_url":"https://api.github.com/repos/parejkoj/fgcmcal/branches{/branch}","tags_url":"https://api.github.com/repos/parejkoj/fgcmcal/tags","blobs_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/refs{/sha}","trees_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/trees{/sha}","statuses_url":"https://api.github.com/repos/parejkoj/fgcmcal/statuses/{sha}","languages_url":"https://api.github.com/repos/parejkoj/fgcmcal/languages","stargazers_url":"https://api.github.com/repos/parejkoj/fgcmcal/stargazers","contributors_url":"https://api.github.com/repos/parejkoj/fgcmcal/contributors","subscribers_url":"https://api.github.com/repos/parejkoj/fgcmcal/subscribers","subscription_url":"https://api.github.com/repos/parejkoj/fgcmcal/subscription","commits_url":"https://api.github.com/repos/parejkoj/fgcmcal/commits{/sha}","git_commits_url":"https://api.github.com/repos/parejkoj/fgcmcal/git/commits{/sha}","comments_url":"https://api.github.com/repos/parejkoj/fgcmcal/comments{/number}","issue_comment_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues/comments{/number}","contents_url":"https://api.github.com/repos/parejkoj/fgcmcal/contents/{+path}","compare_url":"https://api.github.com/repos/parejkoj/fgcmcal/compare/{base}...{head}","merges_url":"https://api.github.com/repos/parejkoj/fgcmcal/merges","archive_url":"https://api.github.com/repos/parejkoj/fgcmcal/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/parejkoj/fgcmcal/downloads","issues_url":"https://api.github.com/repos/parejkoj/fgcmcal/issues{/number}","pulls_url":"https://api.github.com/repos/parejkoj/fgcmcal/pulls{/number}","milestones_url":"https://api.github.com/repos/parejkoj/fgcmcal/milestones{/number}","notifications_url":"https://api.github.com/repos/parejkoj/fgcmcal/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/parejkoj/fgcmcal/labels{/name}","releases_url":"https://api.github.com/repos/parejkoj/fgcmcal/releases{/id}","deployments_url":"https://api.github.com/repos/parejkoj/fgcmcal/deployments","created_at":"2018-11-07T23:15:22Z","updated_at":"2018-11-07T23:15:24Z","pushed_at":"2018-11-14T01:46:48Z","git_url":"git://github.com/parejkoj/fgcmcal.git","ssh_url":"git@github.com:parejkoj/fgcmcal.git","clone_url":"https://github.com/parejkoj/fgcmcal.git","svn_url":"https://github.com/parejkoj/fgcmcal","homepage":"","size":691,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1"},"html":{"href":"https://github.com/parejkoj/fgcmcal/pull/1"},"issue":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/issues/1"},"comments":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/parejkoj/fgcmcal/statuses/49c6ef5ccbb9ecaf5267580af125f1144a61f227"}},"author_association":"OWNER"}}
{"actor":{"display_login":"erykoff"}}