{"action":"created","comment":{"url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/comments/49411941","id":49411941,"diff_hunk":"@@ -0,0 +1,712 @@\n+package org.broadinstitute.hellbender.utils.hdf5;\n+\n+import com.google.api.client.repackaged.com.google.common.annotations.VisibleForTesting;\n+import com.google.common.collect.Sets;\n+import htsjdk.samtools.util.IOUtil;\n+import org.apache.commons.collections4.list.SetUniqueList;\n+import org.apache.commons.lang3.tuple.ImmutablePair;\n+import org.apache.commons.lang3.tuple.Pair;\n+import org.apache.commons.math3.linear.Array2DRowRealMatrix;\n+import org.apache.commons.math3.linear.DefaultRealMatrixChangingVisitor;\n+import org.apache.commons.math3.linear.RealMatrix;\n+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;\n+import org.apache.commons.math3.stat.descriptive.rank.Median;\n+import org.apache.commons.math3.stat.descriptive.rank.Percentile;\n+import org.apache.logging.log4j.LogManager;\n+import org.apache.logging.log4j.Logger;\n+import org.apache.spark.api.java.JavaSparkContext;\n+import org.broadinstitute.hellbender.exceptions.UserException;\n+import org.broadinstitute.hellbender.tools.exome.*;\n+import org.broadinstitute.hellbender.utils.MathUtils;\n+import org.broadinstitute.hellbender.utils.Utils;\n+import org.broadinstitute.hellbender.utils.param.ParamUtils;\n+import org.broadinstitute.hellbender.utils.svd.SVD;\n+import org.broadinstitute.hellbender.utils.svd.SVDFactory;\n+\n+import java.io.File;\n+import java.io.IOException;\n+import java.util.*;\n+import java.util.function.Supplier;\n+import java.util.stream.Collectors;\n+import java.util.stream.DoubleStream;\n+import java.util.stream.IntStream;\n+\n+/**\n+ * This class writes HDF5 PoN files.\n+ */\n+final public class HDF5PoNCreator {\n+\n+    public static final double CURRENT_PON_VERSION = 5.0;\n+\n+    public static final double JOLLIFES_RULE_MEAN_FACTOR = 0.7;\n+\n+    private static final double INV_LN_2 = 1.0 / Math.log(2);\n+\n+    public static final double EPSILON = 10e-10;\n+\n+    private static final Logger logger = LogManager.getLogger(HDF5PoNCreator.class);\n+\n+    private HDF5PoNCreator() {}\n+\n+    /**\n+     * Create an HDF5 PoN file with the output from {@link CombineReadCounts}.\n+     *\n+     * When using this method in testing, see {@link CreatePanelOfNormals} for good default values.\n+     *\n+     * @param ctx If no spark context is available, specify {@code null}\n+     * @param inputPCovFile  pcov file from {@link CombineReadCounts} with samples as columns.  Never {@code null}\n+     * @param numberOfEigenSamples  desired number of eigen samples in the final PoN.  If missing, will apply a rule to determine.\n+     * @param sampleNameBlacklist  sample names in {@code inputPCovFile} that should be ignored.  Names must match exactly.  Never {@code null}.  Entires in this parameter that are not in {@code inputPCovFile} are ignored.\n+     * @param outputHDF5Filename  final filename for the output PoN HDF5 file\n+     * @param isDryRun  whether this is a dry run.  If you wish to do diagnostics and skip the actual writing of a PoN file, set this to true.\n+     * @param initialTargets  the set of targets that was used to generate the {@code inputPCovFile}.\n+     * @param targetFactorPercentileThreshold  The percentile to of extreme target factor values to cut.\n+     * @param extremeColumnMedianCountPercentileThreshold Percentile to cut columns that after they are ranked by median value\n+     * @param countTruncatePercentile percentile (on either end) to truncate extreme values (i.e. extreme high or low values will be set to the high or low count truncation value, which is based on this percentlie)\n+     * @param maximumPercentageZeroTargets  the maximum percentage zero values in a target (across samples) before the target is filtered.\n+     * @param maximumPercentageZeroColumns  the maximum percentage zero values in a sample (across targets) before the sample is filtered.\n+     */\n+    public static void createPoN(final JavaSparkContext ctx, final File inputPCovFile, final OptionalInt numberOfEigenSamples,\n+                                 final List<String> sampleNameBlacklist, final File outputHDF5Filename,\n+                                 final boolean isDryRun, final TargetCollection<Target> initialTargets,\n+                                 final double targetFactorPercentileThreshold,\n+                                 final double extremeColumnMedianCountPercentileThreshold,\n+                                 final double countTruncatePercentile, final double maximumPercentageZeroTargets,\n+                                 final double maximumPercentageZeroColumns) {\n+\n+        Utils.nonNull(inputPCovFile);\n+        ParamUtils.inRange(targetFactorPercentileThreshold, 0, 100, \"Target factor percentile threshold must be in range [0, 100]\");\n+        ParamUtils.inRange(maximumPercentageZeroTargets, 0, 100, \"Maximum percentage of zero-targets must be in range [0, 100]\");\n+        ParamUtils.inRange(maximumPercentageZeroColumns, 0, 100, \"Maximum percentage of zero-columns must be in range [0, 100]\");\n+        ParamUtils.inRange(extremeColumnMedianCountPercentileThreshold, 0, 50, \"Extreme column median percentile threshold must be in range [0, 50]\");\n+        ParamUtils.inRange(countTruncatePercentile, 0, 50, \"Count truncation threshold percentile threshold must be in range [0, 50]\");\n+        Utils.nonNull(sampleNameBlacklist, \"Blacklist sample list is null.  Please use empty list, instead.\");\n+        Utils.nonNull(initialTargets, \"Target collection is null, which is invalid.\");\n+\n+        // Remove low coverage targets:\n+        final Pair<ReadCountCollection, double[]> inputSubsetByUsableTargets = subsetReadCountsToUsableTargets(readReadCountsFromFile(inputPCovFile, initialTargets), targetFactorPercentileThreshold, logger);\n+\n+        // Remove samples listed on the blacklist\n+        ReadCountCollection readCounts = inputSubsetByUsableTargets.getLeft();\n+        readCounts = readCounts.subsetColumns(Sets.difference(new HashSet<>(readCounts.columnNames()), new HashSet<>(sampleNameBlacklist)) );\n+        final double[] targetFactors = inputSubsetByUsableTargets.getRight();\n+\n+        // Normalize read-counts by removing the target factor component:\n+        normalizeReadCountsByTargetFactors(readCounts, targetFactors);\n+\n+        // Create and write the first part of the PoN HDF5 file.\n+        if (!isDryRun) {\n+            writeTargetFactorNormalizeReadCountsAndTargetFactors(outputHDF5Filename, readCounts, targetFactors, initialTargets.targets());\n+        }\n+\n+        // Remove column and targets with too many zeros and targets with\n+        // extreme median coverage:\n+        final int maximumColumnZerosCount = calculateMaximumZerosCount(readCounts.targets().size(), maximumPercentageZeroColumns);\n+        final int maximumTargetZerosCount = calculateMaximumZerosCount(readCounts.columnNames().size(), maximumPercentageZeroTargets);\n+        readCounts = removeColumnsWithTooManyZeros(readCounts, maximumColumnZerosCount, logger);\n+        readCounts = removeTargetsWithTooManyZeros(readCounts, maximumTargetZerosCount, logger);\n+        readCounts = removeColumnsWithExtremeMedianCounts(readCounts, extremeColumnMedianCountPercentileThreshold, logger);\n+\n+        // Impute zero counts to be same as the median for the same target.\n+        // This happens in-place.\n+        imputeZerosCounts(readCounts, logger);\n+\n+        // Truncate extreme count values.\n+        // This happens in-place.\n+        truncateExtremeCounts(readCounts, countTruncatePercentile, logger);\n+\n+        // Normalized by the median and log scale the read counts.\n+        normalizeAndLogReadCounts(readCounts, logger);\n+        subtractBGSCenter(readCounts, logger);\n+\n+        final ReductionResult reduction = calculateReducedPanelAndPInverses(readCounts, numberOfEigenSamples, logger, ctx);\n+\n+        // Write the log-normals and reduced panel matrices.\n+        if (!isDryRun) {\n+            writeLogNormalsReducedPanel(outputHDF5Filename, readCounts, reduction, ctx);\n+        }\n+    }\n+\n+    // Writes normalized and reduced panel matrices to the output HDF5 file.  Also, writes the target variance...\n+    private static void writeLogNormalsReducedPanel(final File outFile, final ReadCountCollection readCounts, final ReductionResult reduction, final JavaSparkContext ctx) {\n+        try (final HDF5File file = new HDF5File(outFile, HDF5File.OpenMode.READ_WRITE)) {\n+            final HDF5PoN pon = new HDF5PoN(file);\n+            pon.setPanelSampleNames(readCounts.columnNames());\n+            pon.setPanelTargetNames(readCounts.targets().stream()\n+                    .map(Target::getName)\n+                    .collect(Collectors.toList()));\n+            pon.setLogNormalCounts(readCounts.counts());\n+            pon.setLogNormalPInverseCounts(reduction.getPseudoInverse());\n+            pon.setReducedPanelCounts(reduction.getReducedCounts());\n+            pon.setReducedPanelPInverseCounts(reduction.getReducedInverse());\n+            pon.setPanelTargets(readCounts.targets());\n+            pon.setVersion(CURRENT_PON_VERSION);\n+            logger.info(\"Calculating and writing the target variances...\");\n+            final double[] targetVariances = createTargetVariance(pon, ctx);\n+            pon.setTargetVariances(targetVariances);\n+        }\n+    }\n+\n+    private static void writeTargetFactorNormalizeReadCountsAndTargetFactors(final File outFile, final ReadCountCollection readCounts, final double[] targetFactors, final List<Target> rawTargets) {\n+        final List<String> sampleNames = readCounts.columnNames();\n+        final List<String> targetNames = readCounts.targets().stream()\n+                .map(Target::getName)\n+                .collect(Collectors.toList());\n+        try (final HDF5File file = new HDF5File(outFile, HDF5File.OpenMode.CREATE)) {\n+            final HDF5PoN pon = new HDF5PoN(file);\n+            pon.setSampleNames(sampleNames);\n+            pon.setTargetNames(targetNames);\n+            pon.setTargetFactors(new Array2DRowRealMatrix(targetFactors));\n+            pon.setNormalCounts(readCounts.counts());\n+            pon.setTargets(readCounts.targets());\n+            pon.setRawTargets(rawTargets);\n+            pon.setRawTargetNames(rawTargets.stream().map(Target::getName).collect(Collectors.toList()));\n+        }\n+    }\n+\n+    /**\n+     *  Creates a new PoN file using values from a given PoN file, except for reduction.  Does the reduction step a\n+     *   second time and populates a new PoN file.\n+     *\n+     * Please note that this code will redo the reduction even if the number of eigensamples is the same in both the input and desired output.\n+     *\n+     * @param ctx  {@code null} is okay if not using Spark\n+     * @param newNumberOfEigenSamples  desired number of eigensamples in the new PoN reduction.\n+     * @param inputHDF5Filename  input PoN file\n+     * @param outputHDF5Filename  output PoN file, which will be the same as the input, except for fields regarding the PoN reduction\n+     */\n+    public static void redoReduction(final JavaSparkContext ctx, final OptionalInt newNumberOfEigenSamples, final File inputHDF5Filename, final File outputHDF5Filename) {\n+        if (inputHDF5Filename.getAbsolutePath().equals(outputHDF5Filename.getAbsolutePath())) {\n+            throw new UserException.CouldNotCreateOutputFile(outputHDF5Filename, \"Cannot create a new PoN overwriting an old one.\");\n+        }\n+\n+        Utils.regularReadableUserFile(inputHDF5Filename);\n+\n+        try (final HDF5File ponReader = new HDF5File(inputHDF5Filename, HDF5File.OpenMode.READ_ONLY)) {\n+            final PoN inputPoN = new HDF5PoN(ponReader);\n+            final ReadCountCollection logNormalizedCounts = new ReadCountCollection(SetUniqueList.setUniqueList(inputPoN.getPanelTargets()), SetUniqueList.setUniqueList(new ArrayList<>(inputPoN.getPanelSampleNames())), inputPoN.getLogNormalizedCounts());\n+            final ReadCountCollection coverageProfile = new ReadCountCollection(SetUniqueList.setUniqueList(inputPoN.getTargets()), SetUniqueList.setUniqueList(new ArrayList<>(inputPoN.getSampleNames())), inputPoN.getNormalizedCounts());\n+\n+            final ReductionResult newReduction = calculateReducedPanelAndPInverses(logNormalizedCounts, newNumberOfEigenSamples, logger, ctx);\n+\n+            final PoN newPoN = new RamPoN(inputPoN, newReduction);\n+            writeTargetFactorNormalizeReadCountsAndTargetFactors(outputHDF5Filename, coverageProfile, newPoN.getTargetFactors().getColumn(0), inputPoN.getRawTargets());\n+            writeLogNormalsReducedPanel(outputHDF5Filename, logNormalizedCounts, newReduction, ctx);\n+        }\n+    }\n+\n+    /**\n+     * SVD and Pseudo inverse calculation.\n+     *\n+     * @param logNormals the input counts for the SVD and reduction steps, fully normalized and already logged.\n+     * @param requestedNumberOfEigenSamples user requested number of eigen samples for the reduced panel.\n+     * @return never {@code null}.\n+     */\n+    @VisibleForTesting\n+    static ReductionResult calculateReducedPanelAndPInverses(final ReadCountCollection logNormals, final OptionalInt requestedNumberOfEigenSamples, final Logger logger, final JavaSparkContext ctx) {\n+\n+        if (ctx == null) {\n+            logger.warn(\"No Spark context provided, not going to use Spark...\");\n+        }\n+\n+        final RealMatrix logNormalCounts = logNormals.counts();\n+        final int numberOfCountColumns = logNormalCounts.getColumnDimension();\n+\n+        logger.info(\"Starting the SVD decomposition of the log-normal counts ...\");\n+        final long svdStartTime = System.currentTimeMillis();\n+        final SVD logNormalsSVD = SVDFactory.createSVD(logNormals.counts(), ctx);\n+        final long svdEndTime = System.currentTimeMillis();\n+        logger.info(String.format(\"Finished the SVD decomposition of the log-normal counts. Elapse of %d seconds\", (svdEndTime - svdStartTime) / 1000));\n+\n+        final int numberOfEigenSamples = determineNumberOfEigenSamples(requestedNumberOfEigenSamples, numberOfCountColumns, logNormalsSVD, logger);\n+        logger.info(String.format(\"Including %d eigen samples in the reduced PoN\", numberOfEigenSamples));\n+        final RealMatrix logNormalsV = logNormalsSVD.getV();\n+        final RealMatrix logNormalsEigenV = logNormalsV.getSubMatrix(0, numberOfCountColumns - 1, 0, numberOfEigenSamples - 1);\n+\n+        final RealMatrix reducedCounts = logNormalCounts.multiply(logNormalsEigenV);\n+\n+        // The Pseudo inverse comes nearly for free from having run the SVD decomposition.\n+        final RealMatrix logNormalsPseudoInverse = logNormalsSVD.getPinv();\n+\n+        logger.info(\"Calculating the reduced PoN inverse matrix...\");\n+        final long riStartTime = System.currentTimeMillis();\n+        final RealMatrix reducedCountsInverse = SVDFactory.createSVD(reducedCounts, ctx).getPinv();\n+        final long riEndTime = System.currentTimeMillis();\n+        logger.info(String.format(\"Finished calculating the reduced PoN inverse matrix. Elapse of %d seconds\", (riEndTime - riStartTime) / 1000));\n+        return new ReductionResult(logNormalsPseudoInverse, reducedCounts, reducedCountsInverse, logNormalsSVD.getSingularValues());\n+    }\n+\n+    /**\n+     * Calculate the number of eigen sample given the user preferences and the\n+     * result of the log-normals SVD.\n+     *\n+     * @param requestedNumberOfEigenSamples the user requested eigen values (empty means the user didn't specify any in particular).\n+     * @param numberOfCountColumns number of count columns in the original input.\n+     * @param logNormalsSVD SVD results on the log-normal counts.\n+     * @return always greater than 0.\n+     */\n+    @VisibleForTesting\n+    static int determineNumberOfEigenSamples(final OptionalInt requestedNumberOfEigenSamples, final int numberOfCountColumns, final SVD logNormalsSVD, final Logger logger) {\n+        final int numberOfEigenSamples;\n+        if (requestedNumberOfEigenSamples.isPresent()) {\n+            if (requestedNumberOfEigenSamples.getAsInt() > numberOfCountColumns) {\n+                logger.warn(String.format(\"The number of requested eigen samples (%d) is larger than the available number of read count columns after filtering (%d), thus we will have to use the latter.\", requestedNumberOfEigenSamples.getAsInt(), numberOfCountColumns));\n+            }\n+            numberOfEigenSamples = Math.min(requestedNumberOfEigenSamples.getAsInt(), numberOfCountColumns);\n+        } else {\n+            final double[] singularValues = logNormalsSVD.getSingularValues();\n+            final double mean = MathUtils.mean(singularValues, 0, singularValues.length);\n+            final double eigenValueCutoff = mean * JOLLIFES_RULE_MEAN_FACTOR; // Jollife's less strict version of Kaiser' Rule.\n+            numberOfEigenSamples = (int) DoubleStream.of(singularValues).filter(v -> v > eigenValueCutoff).count();\n+            logger.info(String.format(\"Jollife's rule produced %d eigen samples out of %d possibles for the reduced PoN\", numberOfEigenSamples, numberOfCountColumns));\n+        }\n+        return numberOfEigenSamples;\n+    }\n+\n+    /**\n+     * Calculates the median of column medians and subtract it from all counts.\n+     * @param readCounts the input counts to center.\n+     * @return the median of medians that has been subtracted from all counts.\n+     */\n+    @VisibleForTesting\n+    static double subtractBGSCenter(final ReadCountCollection readCounts, final Logger logger) {\n+        final RealMatrix counts = readCounts.counts();\n+        final int columnCount = counts.getColumnDimension();\n+        final double[] columnMedians = new double[columnCount];\n+        final double[][] data = counts.getData();\n+        final Median medianCalculator = new Median();\n+        final double[] columnData = new double[data.length];\n+        for (int i = 0; i < columnCount; i++) {\n+            for (int j = 0; j < data.length; j++) {\n+                columnData[j] = data[j][i];\n+            }\n+            columnMedians[i] = medianCalculator.evaluate(columnData);\n+        }\n+        final double medianOfMedians = medianCalculator.evaluate(columnMedians);\n+        counts.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {\n+            @Override\n+            public double visit(final int row, final int column, final double value) {\n+                return value - medianOfMedians;\n+            }\n+        });\n+        logger.info(String.format(\"Counts centered around the BGS center %.2f\", medianOfMedians));\n+        return medianOfMedians;\n+    }\n+\n+    /**\n+     * Final pre-panel normalization that consists on dividing all counts by the median of\n+     * its column and log it with base 2.\n+     * <p>\n+     *     The normalization occurs in-place.\n+     * </p>\n+     *\n+     * @param readCounts the input counts to normalize.\n+     */\n+    @VisibleForTesting\n+    static void normalizeAndLogReadCounts(final ReadCountCollection readCounts, final Logger logger) {\n+        final RealMatrix counts = readCounts.counts();\n+        final int columnCount = counts.getColumnDimension();\n+        final Median medianCalculator = new Median();\n+        for (int i = 0; i < columnCount; i++) {\n+            final double[] colCounts = counts.getColumn(i);\n+            final double median = medianCalculator.evaluate(colCounts);\n+            final double invMedian = 1.0 / median;\n+            for (int j = 0; j < colCounts.length; j++) {\n+                colCounts[j] = Math.log(Math.max(EPSILON, colCounts[j] * invMedian)) * INV_LN_2;\n+            }\n+            counts.setColumn(i, colCounts);\n+        }\n+        logger.info(\"Counts normalized by the column median and changed into bits.\");\n+    }\n+\n+\n+    /**\n+     * Impute zero counts to the median of non-zero values in the enclosing target row.\n+     *\n+     * <p>The imputation is done in-place, thus the input matrix is well be modified as a result of this call.</p>\n+     *\n+     * @param readCounts the input and output read-count matrix.\n+     */\n+    @VisibleForTesting\n+    static void imputeZerosCounts(final ReadCountCollection readCounts, final Logger logger) {\n+\n+        final RealMatrix counts = readCounts.counts();\n+        final int targetCount = counts.getRowDimension();\n+        final int columnCount = counts.getColumnDimension();\n+\n+        // Declared once outside loop to reuse across rows:\n+        final int[] zeroIndexes = new int[columnCount];\n+        final double[] nonZeroValues = new double[columnCount];\n+        final Median medianCalculator = new Median();\n+        int totalCounts = 0;\n+        int totalZeroCounts = 0;\n+\n+        // for each row aka target:\n+        for (int i = 0; i < targetCount; i++) {\n+            // We classify columns as having a zero or not.\n+            // we accumulate nonZero values in nonZeroValues to calculate the median\n+            // where as we accumulate zero indexes that need to be updated later.\n+            final double[] rowCounts = counts.getRow(i);\n+            int zeroCount = 0;\n+            int nonZeroCount = 0;\n+            for (int j = 0; j <  rowCounts.length; j++) {\n+                if (rowCounts[j] != 0.0) {\n+                    nonZeroValues[nonZeroCount++] = rowCounts[j];\n+                } else {\n+                    zeroIndexes[zeroCount++] = j;\n+                }\n+            }\n+            totalCounts += rowCounts.length;\n+            if (zeroCount > 0) { // We only have to do anything if if there is any zeros in the row.\n+                final double nonZeroMedian = medianCalculator.evaluate(nonZeroValues, 0, nonZeroCount);\n+                for (int j = 0; j < zeroCount; j++) {\n+                    counts.setEntry(i, zeroIndexes[j], nonZeroMedian);\n+                }\n+                totalZeroCounts += zeroCount;\n+            }\n+        }\n+        if (totalZeroCounts > 0) {\n+            logger.info(String.format(\"Some 0.0 counts, %d of %d (%.2f%%), where imputed to their enclosing target non-zero median\", totalZeroCounts, totalZeroCounts, 100.0 * (totalZeroCounts / totalCounts)));\n+        } else {\n+            logger.info(\"No count is 0.0 thus no count needed to be imputed.\");\n+        }\n+    }\n+    /**\n+     * Truncates the extreme count values in the input read-count collection.\n+     * Values are forced to be bound by the percentile indicated with the input {@code percentile} which must be\n+     * in the range [0 .. 50.0]. Values under that percentile and the complementary (1 - percentile) are set to the\n+     * corresponding threshold value.\n+     *\n+     * <p>The imputation is done in-place, thus the input matrix is well be modified as a result of this call.</p>\n+     *\n+     * @param readCounts the input and output read-count matrix.\n+     */\n+    @VisibleForTesting\n+    static void truncateExtremeCounts(final ReadCountCollection readCounts, final double percentile, final Logger logger) {\n+\n+        final RealMatrix counts = readCounts.counts();\n+        final int targetCount = counts.getRowDimension();\n+        final int columnCount = counts.getColumnDimension();\n+        final double[] values = new double[targetCount * columnCount];\n+        for (int i = 0; i < targetCount; i++) {\n+            final double[] rowCounts = counts.getRow(i);\n+            System.arraycopy(rowCounts, 0, values, i * columnCount, columnCount);\n+        }\n+        final Percentile bottomPercentileEvaluator = new Percentile(percentile);\n+        final Percentile topPercentileEvaluator = new Percentile(100.0 - percentile);\n+        final double bottomPercentileThreshold = bottomPercentileEvaluator.evaluate(values);\n+        final double topPercentileThreshold = topPercentileEvaluator.evaluate(values);\n+        long totalCounts = 0;\n+        long bottomTruncatedCounts = 0;\n+        long topTruncatedCounts = 0;\n+\n+        for (int i = 0; i < targetCount; i++) {\n+            final double[] rowCounts = counts.getRow(i);\n+            for (int j = 0; j < columnCount; j++) {\n+                final double count = rowCounts[j];\n+                totalCounts++;\n+                if (count < bottomPercentileThreshold) {\n+                    counts.setEntry(i, j, bottomPercentileThreshold);\n+                    bottomTruncatedCounts++;\n+                } else if (count > topPercentileThreshold) {\n+                    counts.setEntry(i, j, topPercentileThreshold);\n+                    topTruncatedCounts++;\n+                }\n+            }\n+        }\n+        if (topTruncatedCounts == 0 && bottomTruncatedCounts == 0) {\n+            logger.info(String.format(\"None of the %d counts where truncated as they all fall in the non-extreme range [%.2f, %.2f]\", totalCounts, bottomPercentileThreshold, topPercentileThreshold));\n+        } else {\n+            final double truncatedPercentage = ((double)(topTruncatedCounts + bottomTruncatedCounts) / totalCounts) * 100;\n+            logger.info(String.format(\"Some counts, %d out of %d (%.2f%%), where truncated as they fall out of the non-extreme range [%.2f, %.2f]\",\n+                    topTruncatedCounts + bottomTruncatedCounts, totalCounts, truncatedPercentage, bottomPercentileThreshold, topPercentileThreshold));\n+        }\n+    }\n+\n+    /**\n+     * Creates a new read-count collection that is a copy of the input but\n+     * dropping columns with extreme median coverage.\n+     *\n+     * @param readCounts the input read-counts.\n+     * @param extremeColumnMedianCountPercentileThreshold bottom percentile to use as a exclusion threshold.\n+     * @return never {@code null}. It might be a reference to the input read-counts if\n+     * there is no columns to be dropped.\n+     */\n+    @VisibleForTesting\n+    static ReadCountCollection removeColumnsWithExtremeMedianCounts(final ReadCountCollection readCounts, final double extremeColumnMedianCountPercentileThreshold, final Logger logger) {\n+        final List<String> columnNames = readCounts.columnNames();\n+        final double[] columnMedians = new double[columnNames.size()];\n+        final RealMatrix counts = readCounts.counts();\n+        final Median medianCalculator = new Median();\n+        for (int i = 0; i < columnMedians.length; i++) {\n+            final double[] columnCounts = counts.getColumn(i);\n+            columnMedians[i] = medianCalculator.evaluate(columnCounts);\n+        }\n+\n+        // Calculate thresholds:\n+        final double bottomExtremeThreshold = new Percentile(extremeColumnMedianCountPercentileThreshold).evaluate(columnMedians);\n+        final double topExtremeThreshold = new Percentile(100 - extremeColumnMedianCountPercentileThreshold).evaluate(columnMedians);\n+\n+        // Determine kept and dropped column sets.\n+        final Set<String> columnsToKeep = new HashSet<>(readCounts.columnNames().size());\n+        final Map<String, Double> columnsToDrop = new LinkedHashMap<>((int) (extremeColumnMedianCountPercentileThreshold * 4.0 / 100.0));\n+        for (int i = 0; i < columnMedians.length; i++) {\n+            if (columnMedians[i] >= bottomExtremeThreshold && columnMedians[i] <= topExtremeThreshold) {\n+                columnsToKeep.add(columnNames.get(i));\n+            } else {\n+                columnsToDrop.put(columnNames.get(i), columnMedians[i]);\n+            }\n+        }\n+\n+        // log and drop columns if it applies\n+        if (columnsToKeep.isEmpty()) {\n+            throw new UserException.BadInput(\"No column count left after applying the extreme counts outlier filter\");\n+        } else if (columnsToKeep.size() == columnNames.size()) {\n+            logger.info(String.format(\"No column dropped due to extreme counts outside [%.10f, %.10f]\", bottomExtremeThreshold, topExtremeThreshold));\n+            return readCounts;\n+        } else {\n+            logger.info(String.format(\"Some columns dropped, %d of %d, as they are classified as having extreme median counts across targets outside [%.10f, %.10f]: e.g. %s\",\n+                    columnNames.size() - columnsToKeep.size(), columnNames.size(), bottomExtremeThreshold, topExtremeThreshold,\n+                    columnsToDrop.entrySet().stream().limit(10).map(kv -> kv.getKey() + \" (\" + kv.getValue() + \")\").collect(Collectors.joining(\", \"))));\n+            return readCounts.subsetColumns(columnsToKeep);\n+        }\n+\n+    }\n+\n+    /**\n+     * Remove targets that have too many counts equal to 0.\n+     * <p>\n+     *     It will return a copy of the input read-count collection with such targets dropped.\n+     * </p>\n+     *\n+     * @param readCounts the input read counts.\n+     * @param maximumTargetZeros maximum number of counts equal to 0. per target tolerated.\n+     * @return never {@code null}. It might be a reference to the input read-counts if there is\n+     *   is no target to be dropped.\n+     */\n+    @VisibleForTesting\n+    static ReadCountCollection removeTargetsWithTooManyZeros(final ReadCountCollection readCounts, final int maximumTargetZeros, final Logger logger) {\n+        final Set<Target> targetsToKeep = new HashSet<>(readCounts.targets().size());\n+        final RealMatrix counts = readCounts.counts();\n+        for (int i = 0; i < counts.getRowDimension(); i++) {","path":"src/main/java/org/broadinstitute/hellbender/utils/hdf5/HDF5PoNCreator.java","position":491,"original_position":491,"commit_id":"24d5ae2a96071dd27802d526d30fe281773d3112","original_commit_id":"24d5ae2a96071dd27802d526d30fe281773d3112","user":{"login":"LeeTL1220","id":2152339,"avatar_url":"https://avatars.githubusercontent.com/u/2152339?v=3","gravatar_id":"","url":"https://api.github.com/users/LeeTL1220","html_url":"https://github.com/LeeTL1220","followers_url":"https://api.github.com/users/LeeTL1220/followers","following_url":"https://api.github.com/users/LeeTL1220/following{/other_user}","gists_url":"https://api.github.com/users/LeeTL1220/gists{/gist_id}","starred_url":"https://api.github.com/users/LeeTL1220/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeeTL1220/subscriptions","organizations_url":"https://api.github.com/users/LeeTL1220/orgs","repos_url":"https://api.github.com/users/LeeTL1220/repos","events_url":"https://api.github.com/users/LeeTL1220/events{/privacy}","received_events_url":"https://api.github.com/users/LeeTL1220/received_events","type":"User","site_admin":false},"body":"Never mind. \r\n\r\nGoing to simplify, but possibly not the same way.","created_at":"2016-01-12T03:21:01Z","updated_at":"2016-01-12T03:21:01Z","html_url":"https://github.com/broadinstitute/gatk-protected/pull/283#discussion_r49411941","pull_request_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283","_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/comments/49411941"},"html":{"href":"https://github.com/broadinstitute/gatk-protected/pull/283#discussion_r49411941"},"pull_request":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283"}}},"pull_request":{"url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283","id":55419657,"html_url":"https://github.com/broadinstitute/gatk-protected/pull/283","diff_url":"https://github.com/broadinstitute/gatk-protected/pull/283.diff","patch_url":"https://github.com/broadinstitute/gatk-protected/pull/283.patch","issue_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/283","number":283,"state":"open","locked":false,"title":"Issue 217 pon qc -- big","user":{"login":"LeeTL1220","id":2152339,"avatar_url":"https://avatars.githubusercontent.com/u/2152339?v=3","gravatar_id":"","url":"https://api.github.com/users/LeeTL1220","html_url":"https://github.com/LeeTL1220","followers_url":"https://api.github.com/users/LeeTL1220/followers","following_url":"https://api.github.com/users/LeeTL1220/following{/other_user}","gists_url":"https://api.github.com/users/LeeTL1220/gists{/gist_id}","starred_url":"https://api.github.com/users/LeeTL1220/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeeTL1220/subscriptions","organizations_url":"https://api.github.com/users/LeeTL1220/orgs","repos_url":"https://api.github.com/users/LeeTL1220/repos","events_url":"https://api.github.com/users/LeeTL1220/events{/privacy}","received_events_url":"https://api.github.com/users/LeeTL1220/received_events","type":"User","site_admin":false},"body":"There are quite a few changes in this pull request.\r\n\r\n- Added the PoN Arm-Level Event Detector, which runs, by default, when creating a PoN.  This is implemented in the PoNIssueDetector.  This class can be extended to include other PoN QC steps.\r\n- PoNIssueDetector has arm-level event detection in both Spark and non-spark versions.\r\n- CreatePanelOfNormals CLI refactored and made into a slimmer CLI class and created several backend objects (e.g. HDF5PoNCreator )\r\n- CreatePanelOfNormals CLI has flag to skip QC step and create the PoN immediately with the input BAM files.\r\n- CreatePanelOfNormals CLI will produce a blacklist file of the samples that are tagged by the PoNIssueDetector.\r\n- CreatePanelOfNormals CLI will produce a weights file that can be used in PerformSegmentation.\r\n- Default locations of blacklist and weights file can be overridden by the user in CreatePanelOfNormals CLI.\r\n- PerformSegmentation CLI can now take optional weights file.\r\n- Added TangentNormalizer class that can do several tangent normalization operations.  Will usually return an instance of TangentNormalizationResult.  Spark and non-spark workflows are implemented.  \r\n- Added method to TangentNormalizer that can allow tangent normalization of all normals in a PoN into the PoN.  This is useful for arm-level event detection and calculating target variances.  \r\n- Targets in reduced panel are stored in the PoN as String table (virtually -- see below).  \r\n- Raw targets are stored in the PoN (totally unfiltered -- identical to input file during creation).\r\n- The storage of targets allows for future tool to extract the targets (issue 265)\r\n- Added generic ability to store String tables in the HDF5 PoN.  Since string tables are not actually supported (yet) in HDF5, the implementation stores a String array and a number of columns field.  The wrapping code hides this implementation detail.\r\n- Target variances are stored in the PoN.  The weight file that is generated is 1/variance for each target.\r\n- Created HDF5PoNCreator class that handles the PoN creation independently from any front-end.  Can also redo the calculation of the reduced PoN related fields without re-creating the entire PoN.  This class contains both Spark and non-spark workflow.\r\n- PoN.java interface updated to reflect the availability of these new fields.\r\n- Added/extended unit tests for comparison of all fields in PoNs.\r\n- Fixed issue with modifiable/unmodifiable PoN field copies not being honored in implementation.\r\n- Created RamPoN, which is a full PoN that is stored entirely in RAM.  This is used for redo'ing reduction in HDFPoNCreator and is useful for testing.\r\n- NormalizeSomaticReadCounts CLI refactored and made into a slimmer CLI class and created several backend objects (e.g. TangentNormalizer)\r\n- Several more steps in PoN creation are now implemented in Spark (non-spark workflow is preserved and still accessible by end users).  Documentation provided that labels these steps.\r\n- Default non-Spark SVD to ojAlgo implementation.  Note that Apache Commons RealMatrix are still used throughout, so conversion is performed.  Also note that this conversion takes very little time (on 150k x 1374 matrix)\r\n- Some testdata generation matlab scripts are provided to ease changes down the road.  These are not really supported and are meant for future developers.\r\n- Case2PoNTargetMapper was split into separate class, since several steps in PoN creation (e.g. target variances calculation and PoN QC)\r\n- More documentation added.\r\n- ReadCountCollection now serializable to make it play nice with Spark.  Test added to make sure it can be serialized/deserialized.\r\n- Target now serializable to make it play nice with Spark.  Test added to make sure it can be serialized/deserialized.\r\n- Corrected some tests in ParamUtils that would behave incorrectly if given a NaN.\r\n- Added methods for simple file reading/writing in ParamUtils.  \r\n- Change RCBSSegmenter to (optionally) use a weights file and accept a minWidth parameter.  See CBS docs for more info.\r\n- Added utility class to aid in the conversion of Spark distributed and local matrices and Apache RealMatrix.\r\n- SVD now uses ojAlgo for the non-spark implementation of SVD.\r\n- Updated the GATK CBS.R file to accept the weights file.  This was esecially necessary since there is a bug in CBS ``segment`` that will not allow weights=NULL to be specified, even though that is the default value.  This is because it checks whether the parameter is specified and if it is, will crash if given NULL.  (shaking fist at R)\r\n- Added integration tests for CreatePanelOfNormals with/without Spark, with/without QC.\r\n- Added ``assertBasicPoNAssumptions`` to ``CreatePanelOfNormalsIntegrationTest.java`` that tests some basic PoN coherence.\r\n- Created PoNTestUtils, which contains some good utility methods for testing PoN logic in our unit/CLI tests.\r\n- Added testing of RamPoN\r\n- Added testing of segmentation with new weights\r\n- Added tests that test whether TangentNormalizer can properly tangent normalize normals in a PoN.\r\n- Moved many of the CreatePanelOfNormalsIntegrationTest tests into the unit test for HDF5PoNCreator.  Added a few more for the target variances.\r\n- Added relevant test data\r\n- Updated pon ground truth testdata with the new fields.  These were verified through manual means and through other tests (mentioned above) before being integrated into the test files.  Matlab scripts are provided.\r\n\r\n\r\ncloses #275\r\ncloses #264\r\ncloses #247\r\ncloses #218\r\ncloses #217\r\ncloses #214","created_at":"2016-01-08T03:00:11Z","updated_at":"2016-01-12T03:21:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1adefd0876c125954289af9871dd7a68578aac57","assignee":{"login":"LeeTL1220","id":2152339,"avatar_url":"https://avatars.githubusercontent.com/u/2152339?v=3","gravatar_id":"","url":"https://api.github.com/users/LeeTL1220","html_url":"https://github.com/LeeTL1220","followers_url":"https://api.github.com/users/LeeTL1220/followers","following_url":"https://api.github.com/users/LeeTL1220/following{/other_user}","gists_url":"https://api.github.com/users/LeeTL1220/gists{/gist_id}","starred_url":"https://api.github.com/users/LeeTL1220/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeeTL1220/subscriptions","organizations_url":"https://api.github.com/users/LeeTL1220/orgs","repos_url":"https://api.github.com/users/LeeTL1220/repos","events_url":"https://api.github.com/users/LeeTL1220/events{/privacy}","received_events_url":"https://api.github.com/users/LeeTL1220/received_events","type":"User","site_admin":false},"milestone":{"url":"https://api.github.com/repos/broadinstitute/gatk-protected/milestones/6","html_url":"https://github.com/broadinstitute/gatk-protected/milestones/CNV%20Tools%20beta%20release","labels_url":"https://api.github.com/repos/broadinstitute/gatk-protected/milestones/6/labels","id":1392068,"number":6,"title":"CNV Tools beta release","description":"This will be the next iteration of tools.  Including:\r\n\r\n- allelic balance integration\r\n- CNLOH calling\r\n- evaluation of the CNLOH and allelic integration\r\n- WGS capabilities (evaluation not required)\r\n- Updated documentation","creator":{"login":"LeeTL1220","id":2152339,"avatar_url":"https://avatars.githubusercontent.com/u/2152339?v=3","gravatar_id":"","url":"https://api.github.com/users/LeeTL1220","html_url":"https://github.com/LeeTL1220","followers_url":"https://api.github.com/users/LeeTL1220/followers","following_url":"https://api.github.com/users/LeeTL1220/following{/other_user}","gists_url":"https://api.github.com/users/LeeTL1220/gists{/gist_id}","starred_url":"https://api.github.com/users/LeeTL1220/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeeTL1220/subscriptions","organizations_url":"https://api.github.com/users/LeeTL1220/orgs","repos_url":"https://api.github.com/users/LeeTL1220/repos","events_url":"https://api.github.com/users/LeeTL1220/events{/privacy}","received_events_url":"https://api.github.com/users/LeeTL1220/received_events","type":"User","site_admin":false},"open_issues":20,"closed_issues":18,"state":"open","created_at":"2015-11-03T19:39:25Z","updated_at":"2016-01-12T00:51:08Z","due_on":"2016-01-29T05:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283/commits","review_comments_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283/comments","review_comment_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/comments{/number}","comments_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/283/comments","statuses_url":"https://api.github.com/repos/broadinstitute/gatk-protected/statuses/24d5ae2a96071dd27802d526d30fe281773d3112","head":{"label":"broadinstitute:issue-217-pon_qc","ref":"issue-217-pon_qc","sha":"24d5ae2a96071dd27802d526d30fe281773d3112","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":34413897,"name":"gatk-protected","full_name":"broadinstitute/gatk-protected","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk-protected","description":"GATK4 development of the license-protected part of the toolkit","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk-protected","forks_url":"https://api.github.com/repos/broadinstitute/gatk-protected/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk-protected/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk-protected/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk-protected/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk-protected/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk-protected/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk-protected/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk-protected/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk-protected/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk-protected/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk-protected/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk-protected/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk-protected/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk-protected/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk-protected/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk-protected/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk-protected/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk-protected/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk-protected/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk-protected/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk-protected/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk-protected/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk-protected/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk-protected/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk-protected/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk-protected/releases{/id}","created_at":"2015-04-22T20:24:45Z","updated_at":"2016-01-11T17:09:39Z","pushed_at":"2016-01-12T00:51:10Z","git_url":"git://github.com/broadinstitute/gatk-protected.git","ssh_url":"git@github.com:broadinstitute/gatk-protected.git","clone_url":"https://github.com/broadinstitute/gatk-protected.git","svn_url":"https://github.com/broadinstitute/gatk-protected","homepage":null,"size":59355,"stargazers_count":4,"watchers_count":4,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":98,"forks":1,"open_issues":98,"watchers":4,"default_branch":"master"}},"base":{"label":"broadinstitute:master","ref":"master","sha":"48b85fc033663d80e5d0cf8c260ba55a86bd2c3d","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":34413897,"name":"gatk-protected","full_name":"broadinstitute/gatk-protected","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk-protected","description":"GATK4 development of the license-protected part of the toolkit","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk-protected","forks_url":"https://api.github.com/repos/broadinstitute/gatk-protected/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk-protected/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk-protected/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk-protected/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk-protected/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk-protected/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk-protected/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk-protected/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk-protected/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk-protected/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk-protected/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk-protected/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk-protected/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk-protected/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk-protected/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk-protected/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk-protected/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk-protected/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk-protected/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk-protected/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk-protected/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk-protected/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk-protected/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk-protected/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk-protected/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk-protected/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk-protected/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk-protected/releases{/id}","created_at":"2015-04-22T20:24:45Z","updated_at":"2016-01-11T17:09:39Z","pushed_at":"2016-01-12T00:51:10Z","git_url":"git://github.com/broadinstitute/gatk-protected.git","ssh_url":"git@github.com:broadinstitute/gatk-protected.git","clone_url":"https://github.com/broadinstitute/gatk-protected.git","svn_url":"https://github.com/broadinstitute/gatk-protected","homepage":null,"size":59355,"stargazers_count":4,"watchers_count":4,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":98,"forks":1,"open_issues":98,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283"},"html":{"href":"https://github.com/broadinstitute/gatk-protected/pull/283"},"issue":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/283"},"comments":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/issues/283/comments"},"review_comments":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283/comments"},"review_comment":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/pulls/283/commits"},"statuses":{"href":"https://api.github.com/repos/broadinstitute/gatk-protected/statuses/24d5ae2a96071dd27802d526d30fe281773d3112"}}}}