def createAndCleanTPED(tped, tfam, samples, oldSamples, chosenSamples,
                       prefix, completion, completionT, concordance,
                       concordanceT):
    """Complete a TPED for duplicate samples.

    :param tped: the ``tped`` containing the duplicated samples.
    :param tfam: the ``tfam`` containing the duplicated samples.
    :param samples: the updated position of the samples in the ``tped``
                    containing only duplicated samples.
    :param oldSamples: the original duplicated sample positions.
    :param chosenSamples: the position of the chosen samples.
    :param prefix: the prefix of all the files.
    :param completion: the completion of each of the duplicated samples.
    :param completionT: the completion threshold.
    :param concordance: the pairwise concordance of each of the duplicated
                        samples.
    :param concordanceT: the concordance threshold.

    :type tped: :py:class:`numpy.array`
    :type tfam: :py:class:`numpy.array`
    :type samples: dict
    :type oldSamples: dict
    :type chosenSamples: dict
    :type prefix: str
    :type completion: :py:class:`numpy.array`
    :type completionT: float
    :type concordance: dict
    :type concordanceT: float

    Using a ``tped`` containing duplicated samples, it creates a ``tped``
    containing unique samples by completing a chosen sample with the other
    replicates.

    .. note::
        A chosen sample is not completed using bad replicates (those that
        don't have a concordance or a completion higher than a certain
        threshold). The bad replicates are written in the file
        ``prefix.not_good_enough``.

    """
    zeroedOutFile = None
    try:
        zeroedOutFile = open(prefix + ".zeroed_out", "w")
    except IOError:
        msg = "%(prefix)s.zeroed_out: can't write file" % locals()
        raise ProgramError(msg)
    print >>zeroedOutFile, "\t".join(["famID", "indID", "snpID"])

    notGoodEnoughFile = None
    try:
        notGoodEnoughFile = open(prefix + ".not_good_enough", "w")
    except IOError:
        msg = "%(prefix)s.not_good_enough: can't write file" % locals()
        raise ProgramError(msg)
    print >>notGoodEnoughFile, "\t".join(["origIndex", "dupIndex", "famID",
                                          "indID", "reason"])
    notGoodEnoughSamples = defaultdict(set)

    # Split the tped in 'snpInfo' and 'genotypes'
    snpInfo = tped[:, :4]
    genotypes = tped[:, 4:]

    # Getting the completion and concordance indexes
    completionSampleID = {}
    concordanceSampleID = {}
    for sampleID, indexes in samples.iteritems():
        # Checking the completion
        completionToKeep = np.where(completion[indexes] >= completionT)[0]
        completionToRemove = np.where(completion[indexes] < completionT)[0]
        for k in completionToRemove:
            notGoodEnoughSamples[(str(oldSamples[sampleID][k]+1),
                                  str(indexes[k]+1), sampleID[0],
                                  sampleID[1])].add("completion")

        # Checking the concordance
        concordanceToKeep = np.where(concordance[sampleID] >= concordanceT)[0]
        concordanceToRemove = np.where(
            concordance[sampleID] < concordanceT
        )[0]
        for k in concordanceToRemove:
            notGoodEnoughSamples[(str(oldSamples[sampleID][k]+1),
                                  str(indexes[k]+1), sampleID[0],
                                  sampleID[1])].add("concordance")

        completionSampleID[sampleID] = completionToKeep
        concordanceSampleID[sampleID] = concordanceToKeep

    for i, curSNPgenotypes in enumerate(genotypes):
        for sampleID, indexes in samples.iteritems():
            # Getting the concordance and the completion
            concordanceToKeep = concordanceSampleID[sampleID]
            completionToKeep = completionSampleID[sampleID]

            # Getting the chosen sample index
            chosenOne = chosenSamples[sampleID]

            # Getting the duplicates' genotypes
            curGenotypes = curSNPgenotypes[indexes]

            # Subsetting for the good completion and concordance
            curGenotypes = curGenotypes[np.intersect1d(concordanceToKeep,
                                                       completionToKeep)]

            # Removing the no call from the genotypes
            cleanedCurGenotype = curGenotypes[np.where(curGenotypes != "0 0")]

            # Checking the number of unique genotypes
            uniqueGenotypes = np.unique(cleanedCurGenotype)

            # Checking the number of unique genotypes (except 0 0)
            if len(uniqueGenotypes) > 1:
                # There are more than one unique genotypes (except 0 0)
                # len = 0 means all were 0 0
                # len = 1 means all the same
                # len > 1 means more than one unique genotypes
                possibleAlleles = [
                    set() for k in xrange(len(uniqueGenotypes))
                ]
                for k, geno in enumerate(uniqueGenotypes):
                    possibleAlleles[k] |= set(geno.split(" "))
                allEqual = True
                for k in xrange(len(possibleAlleles)):
                    for l in xrange(k+1, len(possibleAlleles)):
                        if possibleAlleles[k] != possibleAlleles[l]:
                            allEqual = False
                            break
                    if not allEqual:
                        break
                if not allEqual:
                    # Changing the chosen sample's genotype to no call
                    tped[i][chosenOne+4] = "0 0"
                    print >>zeroedOutFile, "\t".join([sampleID[0], sampleID[1],
                                                      snpInfo[i][1]])
            else:
                # Do we want to complete the chosen sample's genotype???
                pass

    # Writing the not good enough file
    for key, value in notGoodEnoughSamples.iteritems():
        print >>notGoodEnoughFile, "\t".join(list(key) + [";".join(value)])

    # Closing the output files
    zeroedOutFile.close()
    notGoodEnoughFile.close()

    # Getting the indexes to keep
    sampleToKeep = np.array(chosenSamples.values()) + 4
    sampleToKeep.sort()
    toKeep = np.append(np.array(range(4)), sampleToKeep)

    # Subsetting the tped
    newTPED = tped[:, toKeep]

    # Getting the indexes of the tfam
    toKeep = []
    for sampleID, indexes in oldSamples.iteritems():
        i = samples[sampleID].index(chosenSamples[sampleID])
        toKeep.append(indexes[i])
    toKeep = np.array(toKeep)
    toKeep.sort()

    # Subsetting the tfam
    newTFAM = tfam[toKeep]

    return newTPED, newTFAM
def createAndCleanTPED ( tped , tfam , samples , oldSamples , chosenSamples , prefix , completion , completionT , concordance , concordanceT ) : zeroedOutFile = None try : zeroedOutFile = open ( prefix + ".zeroed_out" , "w" ) except IOError : msg = "%(prefix)s.zeroed_out: can't write file" % locals ( ) raise ProgramError ( msg ) print >> zeroedOutFile , "\t" . join ( [ "famID" , "indID" , "snpID" ] ) notGoodEnoughFile = None try : notGoodEnoughFile = open ( prefix + ".not_good_enough" , "w" ) except IOError : msg = "%(prefix)s.not_good_enough: can't write file" % locals ( ) raise ProgramError ( msg ) print >> notGoodEnoughFile , "\t" . join ( [ "origIndex" , "dupIndex" , "famID" , "indID" , "reason" ] ) notGoodEnoughSamples = defaultdict ( set ) snpInfo = tped [ : , : 4 ] genotypes = tped [ : , 4 : ] completionSampleID = { } concordanceSampleID = { } for sampleID , indexes in samples . iteritems ( ) : completionToKeep = np . where ( completion [ indexes ] >= completionT ) [ 0 ] completionToRemove = np . where ( completion [ indexes ] < completionT ) [ 0 ] for k in completionToRemove : notGoodEnoughSamples [ ( str ( oldSamples [ sampleID ] [ k ] + 1 ) , str ( indexes [ k ] + 1 ) , sampleID [ 0 ] , sampleID [ 1 ] ) ] . add ( "completion" ) concordanceToKeep = np . where ( concordance [ sampleID ] >= concordanceT ) [ 0 ] concordanceToRemove = np . where ( concordance [ sampleID ] < concordanceT ) [ 0 ] for k in concordanceToRemove : notGoodEnoughSamples [ ( str ( oldSamples [ sampleID ] [ k ] + 1 ) , str ( indexes [ k ] + 1 ) , sampleID [ 0 ] , sampleID [ 1 ] ) ] . add ( "concordance" ) completionSampleID [ sampleID ] = completionToKeep concordanceSampleID [ sampleID ] = concordanceToKeep for i , curSNPgenotypes in enumerate ( genotypes ) : for sampleID , indexes in samples . iteritems ( ) : concordanceToKeep = concordanceSampleID [ sampleID ] completionToKeep = completionSampleID [ sampleID ] chosenOne = chosenSamples [ sampleID ] curGenotypes = curSNPgenotypes [ indexes ] curGenotypes = curGenotypes [ np . intersect1d ( concordanceToKeep , completionToKeep ) ] cleanedCurGenotype = curGenotypes [ np . where ( curGenotypes != "0 0" ) ] uniqueGenotypes = np . unique ( cleanedCurGenotype ) if len ( uniqueGenotypes ) > 1 : possibleAlleles = [ set ( ) for k in xrange ( len ( uniqueGenotypes ) ) ] for k , geno in enumerate ( uniqueGenotypes ) : possibleAlleles [ k ] |= set ( geno . split ( " " ) ) allEqual = True for k in xrange ( len ( possibleAlleles ) ) : for l in xrange ( k + 1 , len ( possibleAlleles ) ) : if possibleAlleles [ k ] != possibleAlleles [ l ] : allEqual = False break if not allEqual : break if not allEqual : tped [ i ] [ chosenOne + 4 ] = "0 0" print >> zeroedOutFile , "\t" . join ( [ sampleID [ 0 ] , sampleID [ 1 ] , snpInfo [ i ] [ 1 ] ] ) else : pass for key , value in notGoodEnoughSamples . iteritems ( ) : print >> notGoodEnoughFile , "\t" . join ( list ( key ) + [ ";" . join ( value ) ] ) zeroedOutFile . close ( ) notGoodEnoughFile . close ( ) sampleToKeep = np . array ( chosenSamples . values ( ) ) + 4 sampleToKeep . sort ( ) toKeep = np . append ( np . array ( range ( 4 ) ) , sampleToKeep ) newTPED = tped [ : , toKeep ] toKeep = [ ] for sampleID , indexes in oldSamples . iteritems ( ) : i = samples [ sampleID ] . index ( chosenSamples [ sampleID ] ) toKeep . append ( indexes [ i ] ) toKeep = np . array ( toKeep ) toKeep . sort ( ) newTFAM = tfam [ toKeep ] return newTPED , newTFAM
Complete a TPED for duplicate samples.

    :param tped: the ``tped`` containing the duplicated samples.
    :param tfam: the ``tfam`` containing the duplicated samples.
    :param samples: the updated position of the samples in the ``tped``
                    containing only duplicated samples.
    :param oldSamples: the original duplicated sample positions.
    :param chosenSamples: the position of the chosen samples.
    :param prefix: the prefix of all the files.
    :param completion: the completion of each of the duplicated samples.
    :param completionT: the completion threshold.
    :param concordance: the pairwise concordance of each of the duplicated
                        samples.
    :param concordanceT: the concordance threshold.

    :type tped: :py:class:`numpy.array`
    :type tfam: :py:class:`numpy.array`
    :type samples: dict
    :type oldSamples: dict
    :type chosenSamples: dict
    :type prefix: str
    :type completion: :py:class:`numpy.array`
    :type completionT: float
    :type concordance: dict
    :type concordanceT: float

    Using a ``tped`` containing duplicated samples, it creates a ``tped``
    containing unique samples by completing a chosen sample with the other
    replicates.

    .. note::
        A chosen sample is not completed using bad replicates (those that
        don't have a concordance or a completion higher than a certain
        threshold). The bad replicates are written in the file
        ``prefix.not_good_enough``.