def cost(self, t_node, branch_length, multiplicity=2.0):
        '''
        returns the cost associated with a branch starting at t_node
        t_node is time before present, the branch goes back in time

        Args:
            - t_node:           time of the node
            - branch_length:    branch length, determines when this branch merges with sister
            - multiplicity:     2 if merger is binary, higher if this is a polytomy
        '''
        merger_time = t_node+branch_length
        return self.integral_merger_rate(merger_time) - self.integral_merger_rate(t_node)\
                 - np.log(self.total_merger_rate(merger_time))*(multiplicity-1.0)/multiplicity

def cost ( self , t_node , branch_length , multiplicity = 2.0 ) : merger_time = t_node + branch_length return self . integral_merger_rate ( merger_time ) - self . integral_merger_rate ( t_node ) - np . log ( self . total_merger_rate ( merger_time ) ) * ( multiplicity - 1.0 ) / multiplicity

returns the cost associated with a branch starting at t_node t_node is time before present the branch goes back in time


def attach_to_tree(self):
        '''
        attaches the the merger cost to each branch length interpolator in the tree.
        '''
        for clade in self.tree.find_clades():
            if clade.up is not None:
                clade.branch_length_interpolator.merger_cost = self.cost

def attach_to_tree ( self ) : for clade in self . tree . find_clades ( ) : if clade . up is not None : clade . branch_length_interpolator . merger_cost = self . cost

attaches the the merger cost to each branch length interpolator in the tree .


def optimize_Tc(self):
        '''
        determines the coalescent time scale that optimizes the coalescent likelihood of the tree
        '''
        from scipy.optimize import minimize_scalar
        initial_Tc = self.Tc
        def cost(Tc):
            self.set_Tc(Tc)
            return -self.total_LH()

        sol = minimize_scalar(cost, bounds=[ttconf.TINY_NUMBER,10.0])
        if "success" in sol and sol["success"]:
            self.set_Tc(sol['x'])
        else:
            self.logger("merger_models:optimze_Tc: optimization of coalescent time scale failed: " + str(sol), 0, warn=True)
            self.set_Tc(initial_Tc.y, T=initial_Tc.x)


def prof2seq(profile, gtr, sample_from_prof=False, normalize=True):
    """
    Convert profile to sequence and normalize profile across sites.

    Parameters
    ----------

     profile : numpy 2D array
        Profile. Shape of the profile should be (L x a), where L - sequence
        length, a - alphabet size.

     gtr : gtr.GTR
        Instance of the GTR class to supply the sequence alphabet

     collapse_prof : bool
        Whether to convert the profile to the delta-function

    Returns
    -------
     seq : numpy.array
        Sequence as numpy array of length L

     prof_values :  numpy.array
        Values of the profile for the chosen sequence characters (length L)

     idx : numpy.array
        Indices chosen from profile as array of length L
    """

    # normalize profile such that probabilities at each site sum to one
    if normalize:
        tmp_profile, pre=normalize_profile(profile, return_offset=False)
    else:
        tmp_profile = profile

    # sample sequence according to the probabilities in the profile
    # (sampling from cumulative distribution over the different states)
    if sample_from_prof:
        cumdis = tmp_profile.cumsum(axis=1).T
        randnum = np.random.random(size=cumdis.shape[1])
        idx = np.argmax(cumdis>=randnum, axis=0)
    else:
        idx = tmp_profile.argmax(axis=1)
    seq = gtr.alphabet[idx]  # max LH over the alphabet

    prof_values = tmp_profile[np.arange(tmp_profile.shape[0]), idx]

    return seq, prof_values, idx

Convert profile to sequence and normalize profile across sites .


def normalize_profile(in_profile, log=False, return_offset = True):
    """return a normalized version of a profile matrix

    Parameters
    ----------
    in_profile : np.array
        shape Lxq, will be normalized to one across each row
    log : bool, optional
        treat the input as log probabilities
    return_offset : bool, optional
        return the log of the scale factor for each row

    Returns
    -------
    tuple
        normalized profile (fresh np object) and offset (if return_offset==True)
    """
    if log:
        tmp_prefactor = in_profile.max(axis=1)
        tmp_prof = np.exp(in_profile.T - tmp_prefactor).T
    else:
        tmp_prefactor = 0.0
        tmp_prof = in_profile

    norm_vector = tmp_prof.sum(axis=1)
    return (np.copy(np.einsum('ai,a->ai',tmp_prof,1.0/norm_vector)),
            (np.log(norm_vector) + tmp_prefactor) if return_offset else None)

def normalize_profile ( in_profile , log = False , return_offset = True ) : if log : tmp_prefactor = in_profile . max ( axis = 1 ) tmp_prof = np . exp ( in_profile . T - tmp_prefactor ) . T else : tmp_prefactor = 0.0 tmp_prof = in_profile norm_vector = tmp_prof . sum ( axis = 1 ) return ( np . copy ( np . einsum ( 'ai,a->ai' , tmp_prof , 1.0 / norm_vector ) ) , ( np . log ( norm_vector ) + tmp_prefactor ) if return_offset else None )


def gtr(self, value):
        """
        Set a new GTR object

        Parameters
        -----------

         value : GTR
            the new GTR object
        """
        if not (isinstance(value, GTR) or isinstance(value, GTR_site_specific)):
            raise TypeError(" GTR instance expected")
        self._gtr = value

def
gtr
(
self
,
value
)
:
if
not
(
isinstance
(
value
,
GTR
)
or
isinstance
(
value
,
GTR_site_specific
)
)
:
raise
TypeError
(
" GTR instance expected"
)
self
.
_gtr
=
value


def set_gtr(self, in_gtr, **kwargs):
        """
        Create new GTR model if needed, and set the model as an attribute of the
        TreeAnc class

        Parameters
        -----------

         in_gtr : str, GTR
            The gtr model to be assigned. If string is passed,
            it is taken as the name of a standard GTR model, and is
            attempted to be created through :code:`GTR.standard()` interface. If a
            GTR instance is passed, it is set directly .

         **kwargs
            Keyword arguments to construct the GTR model. If none are passed, defaults
            are assumed.

        """
        if isinstance(in_gtr, str):
            self._gtr = GTR.standard(model=in_gtr, **kwargs)
            self._gtr.logger = self.logger

        elif isinstance(in_gtr, GTR) or isinstance(in_gtr, GTR_site_specific):
            self._gtr = in_gtr
            self._gtr.logger=self.logger
        else:
            self.logger("TreeAnc.gtr_setter: can't interpret GTR model", 1, warn=True)
            raise TypeError("Cannot set GTR model in TreeAnc class: GTR or "
                "string expected")

        if self._gtr.ambiguous is None:
            self.fill_overhangs=False

Create new GTR model if needed and set the model as an attribute of the TreeAnc class


def seq_len(self,L):
        """set the length of the uncompressed sequence. its inverse 'one_mutation'
        is frequently used as a general length scale. This can't be changed once
        it is set.

        Parameters
        ----------
        L : int
            length of the sequence alignment
        """
        if (not hasattr(self, '_seq_len')) or self._seq_len is None:
            if L:
                self._seq_len = int(L)
        else:
            self.logger("TreeAnc: one_mutation and sequence length can't be reset",1)

set the length of the uncompressed sequence . its inverse one_mutation is frequently used as a general length scale . This can t be changed once it is set .


def _attach_sequences_to_nodes(self):
        '''
        For each node of the tree, check whether there is a sequence available
        in the alignment and assign this sequence as a character array
        '''
        failed_leaves= 0
        if self.is_vcf:
            # if alignment is specified as difference from ref
            dic_aln = self.aln
        else:
            # if full alignment is specified
            dic_aln = {k.name: seq2array(k.seq, fill_overhangs=self.fill_overhangs,
                                                   ambiguous_character=self.gtr.ambiguous)
                                for k in self.aln} #

        # loop over leaves and assign multiplicities of leaves (e.g. number of identical reads)
        for l in self.tree.get_terminals():
            if l.name in self.seq_multiplicity:
                l.count = self.seq_multiplicity[l.name]
            else:
                l.count = 1.0

        # loop over tree, and assign sequences
        for l in self.tree.find_clades():
            if l.name in dic_aln:
                l.sequence= dic_aln[l.name]
            elif l.is_terminal():
                self.logger("***WARNING: TreeAnc._attach_sequences_to_nodes: NO SEQUENCE FOR LEAF: %s" % l.name, 0, warn=True)
                failed_leaves += 1
                l.sequence = seq2array(self.gtr.ambiguous*self.seq_len, fill_overhangs=self.fill_overhangs,
                                                 ambiguous_character=self.gtr.ambiguous)
                if failed_leaves > self.tree.count_terminals()/3:
                    self.logger("ERROR: At least 30\\% terminal nodes cannot be assigned with a sequence!\n", 0, warn=True)
                    self.logger("Are you sure the alignment belongs to the tree?", 2, warn=True)
                    break
            else: # could not assign sequence for internal node - is OK
                pass

        if failed_leaves:
            self.logger("***WARNING: TreeAnc: %d nodes don't have a matching sequence in the alignment."
                        " POSSIBLE ERROR."%failed_leaves, 0, warn=True)

        # extend profile to contain additional unknown characters
        self.extend_profile()
        return self.make_reduced_alignment()

For each node of the tree check whether there is a sequence available in the alignment and assign this sequence as a character array


def prepare_tree(self):
        """
        Set link to parent and calculate distance to root for all tree nodes.
        Should be run once the tree is read and after every rerooting,
        topology change or branch length optimizations.
        """
        self.tree.root.branch_length = 0.001
        self.tree.root.mutation_length = self.tree.root.branch_length
        self.tree.root.mutations = []
        self.tree.ladderize()
        self._prepare_nodes()
        self._leaves_lookup = {node.name:node for node in self.tree.get_terminals()}

def prepare_tree ( self ) : self . tree . root . branch_length = 0.001 self . tree . root . mutation_length = self . tree . root . branch_length self . tree . root . mutations = [ ] self . tree . ladderize ( ) self . _prepare_nodes ( ) self . _leaves_lookup = { node . name : node for node in self . tree . get_terminals ( ) }

Set link to parent and calculate distance to root for all tree nodes . Should be run once the tree is read and after every rerooting topology change or branch length optimizations .


def _prepare_nodes(self):
        """
        Set auxilliary parameters to every node of the tree.
        """
        self.tree.root.up = None
        self.tree.root.bad_branch=self.tree.root.bad_branch if hasattr(self.tree.root, 'bad_branch') else False
        internal_node_count = 0
        for clade in self.tree.get_nonterminals(order='preorder'): # parents first
            internal_node_count+=1
            if clade.name is None:
                clade.name = "NODE_" + format(self._internal_node_count, '07d')
                self._internal_node_count += 1
            for c in clade.clades:
                if c.is_terminal():
                    c.bad_branch = c.bad_branch if hasattr(c, 'bad_branch') else False
                c.up = clade

        for clade in self.tree.get_nonterminals(order='postorder'): # parents first
            clade.bad_branch = all([c.bad_branch for c in clade])

        self._calc_dist2root()
        self._internal_node_count = max(internal_node_count, self._internal_node_count)

def
_prepare_nodes
(
self
)
:
self
.
tree
.
root
.
up
=
self
.
tree
.
root
.
bad_branch
=
self
.
tree
.
root
.
bad_branch
if
hasattr
(
self
.
tree
.
root
,
'bad_branch'
)
else
False
internal_node_count
=
0
for
clade
in
self
.
tree
.
get_nonterminals
(
order
=
'preorder'
)
:
# parents first
internal_node_count
+=
1
if
clade
.
name
is
:
clade
.
name
=
"NODE_"
+
format
(
self
.
_internal_node_count
,
'07d'
)
self
.
_internal_node_count
+=
1
for
c
in
clade
.
clades
:
if
c
.
is_terminal
(
)
:
c
.
bad_branch
=
c
.
bad_branch
if
hasattr
(
c
,
'bad_branch'
)
else
False
c
.
up
=
clade
for
clade
in
self
.
tree
.
get_nonterminals
(
order
=
'postorder'
)
:
# parents first
clade
.
bad_branch
=
all
(
[
c
.
bad_branch
for
c
in
clade
]
)
self
.
_calc_dist2root
(
)
self
.
_internal_node_count
=
max
(
internal_node_count
,
self
.
_internal_node_count
)

Set auxilliary parameters to every node of the tree .


def _calc_dist2root(self):
        """
        For each node in the tree, set its root-to-node distance as dist2root
        attribute
        """
        self.tree.root.dist2root = 0.0
        for clade in self.tree.get_nonterminals(order='preorder'): # parents first
            for c in clade.clades:
                if not hasattr(c, 'mutation_length'):
                    c.mutation_length=c.branch_length
                c.dist2root = c.up.dist2root + c.mutation_length

def
_calc_dist2root
(
self
)
:
self
.
tree
.
root
.
dist2root
=
0.0
for
clade
in
self
.
tree
.
get_nonterminals
(
order
=
'preorder'
)
:
# parents first
for
c
in
clade
.
clades
:
if
not
hasattr
(
c
,
'mutation_length'
)
:
c
.
mutation_length
=
c
.
branch_length
c
.
dist2root
=
c
.
up
.
dist2root
+
c
.
mutation_length

For each node in the tree set its root - to - node distance as dist2root attribute


def reconstruct_anc(self, method='probabilistic', infer_gtr=False,
                        marginal=False, **kwargs):
        """Reconstruct ancestral sequences

        Parameters
        ----------
        method : str
           Method to use. Supported values are "fitch" and "ml"

        infer_gtr : bool
           Infer a GTR model before reconstructing the sequences

        marginal : bool
           Assign sequences that are most likely after averaging over all other nodes
           instead of the jointly most likely sequences.
        **kwargs
            additional keyword arguments that are passed down to :py:meth:`TreeAnc.infer_gtr` and :py:meth:`TreeAnc._ml_anc`

        Returns
        -------
        N_diff : int
           Number of nucleotides different from the previous
           reconstruction.  If there were no pre-set sequences, returns N*L

        """
        self.logger("TreeAnc.infer_ancestral_sequences with method: %s, %s"%(method, 'marginal' if marginal else 'joint'), 1)
        if (self.tree is None) or (self.aln is None):
            self.logger("TreeAnc.infer_ancestral_sequences: ERROR, alignment or tree are missing", 0)
            return ttconf.ERROR

        if method in ['ml', 'probabilistic']:
            if marginal:
                _ml_anc = self._ml_anc_marginal
            else:
                _ml_anc = self._ml_anc_joint
        else:
            _ml_anc = self._fitch_anc

        if infer_gtr:
            tmp = self.infer_gtr(marginal=marginal, **kwargs)
            if tmp==ttconf.ERROR:
                return tmp
            N_diff = _ml_anc(**kwargs)
        else:
            N_diff = _ml_anc(**kwargs)

        return N_diff


def get_branch_mutation_matrix(self, node, full_sequence=False):
        """uses results from marginal ancestral inference to return a joint
        distribution of the sequence states at both ends of the branch.

        Parameters
        ----------
        node : Phylo.clade
            node of the tree
        full_sequence : bool, optional
            expand the sequence to the full sequence, if false (default)
            the there will be one mutation matrix for each column in the
            reduced alignment

        Returns
        -------
        numpy.array
            an Lxqxq stack of matrices (q=alphabet size, L (reduced)sequence length)
        """
        pp,pc = self.marginal_branch_profile(node)

        # calculate pc_i [e^Qt]_ij pp_j for each site
        expQt = self.gtr.expQt(self._branch_length_to_gtr(node))
        if len(expQt.shape)==3: # site specific model
            mut_matrix_stack = np.einsum('ai,aj,ija->aij', pc, pp, expQt)
        else:
            mut_matrix_stack = np.einsum('ai,aj,ij->aij', pc, pp, expQt)

        # normalize this distribution
        normalizer = mut_matrix_stack.sum(axis=2).sum(axis=1)
        mut_matrix_stack = np.einsum('aij,a->aij', mut_matrix_stack, 1.0/normalizer)

        # expand to full sequence if requested
        if full_sequence:
            return mut_matrix_stack[self.full_to_reduced_sequence_map]
        else:
            return mut_matrix_stack

uses results from marginal ancestral inference to return a joint distribution of the sequence states at both ends of the branch .


def expanded_sequence(self, node, include_additional_constant_sites=False):
        """
        Expand a nodes compressed sequence into the real sequence

        Parameters
        ----------
        node : PhyloTree.Clade
           Tree node

        Returns
        -------
        seq : np.array
           Sequence as np.array of chars
        """
        if include_additional_constant_sites:
            L = self.seq_len
        else:
            L = self.seq_len - self.additional_constant_sites

        return node.cseq[self.full_to_reduced_sequence_map[:L]]

def expanded_sequence ( self , node , include_additional_constant_sites = False ) : if include_additional_constant_sites : L = self . seq_len else : L = self . seq_len - self . additional_constant_sites return node . cseq [ self . full_to_reduced_sequence_map [ : L ] ]


def _fitch_anc(self, **kwargs):
        """
        Reconstruct ancestral states using Fitch's algorithm. The method requires
        sequences to be assigned to leaves. It implements the iteration from
        leaves to the root constructing the Fitch profiles for each character of
        the sequence, and then by propagating from the root to the leaves,
        reconstructs the sequences of the internal nodes.

        Keyword Args
        ------------

        Returns
        -------
        Ndiff : int
           Number of the characters that changed since the previous
           reconstruction. These changes are determined from the pre-set
           sequence attributes of the nodes. If there are no sequences available
           (i.e., no reconstruction has been made before), returns the total
           number of characters in the tree.

        """
        # set fitch profiiles to each terminal node

        for l in self.tree.get_terminals():
            l.state = [[k] for k in l.cseq]

        L = len(self.tree.get_terminals()[0].cseq)

        self.logger("TreeAnc._fitch_anc: Walking up the tree, creating the Fitch profiles",2)
        for node in self.tree.get_nonterminals(order='postorder'):
            node.state = [self._fitch_state(node, k) for k in range(L)]

        ambs = [i for i in range(L) if len(self.tree.root.state[i])>1]
        if len(ambs) > 0:
            for amb in ambs:
                self.logger("Ambiguous state of the root sequence "
                                    "in the position %d: %s, "
                                    "choosing %s" % (amb, str(self.tree.root.state[amb]),
                                                     self.tree.root.state[amb][0]), 4)
        self.tree.root.cseq = np.array([k[np.random.randint(len(k)) if len(k)>1 else 0]
                                           for k in self.tree.root.state])

        if self.is_vcf:
            self.tree.root.sequence = self.dict_sequence(self.tree.root)
        else:
            self.tree.root.sequence = self.expanded_sequence(self.tree.root)

        self.logger("TreeAnc._fitch_anc: Walking down the self.tree, generating sequences from the "
                         "Fitch profiles.", 2)
        N_diff = 0
        for node in self.tree.get_nonterminals(order='preorder'):
            if node.up != None: # not root
                sequence =  np.array([node.up.cseq[i]
                        if node.up.cseq[i] in node.state[i]
                        else node.state[i][0] for i in range(L)])
                if hasattr(node, 'sequence'):
                    N_diff += (sequence!=node.cseq).sum()
                else:
                    N_diff += L
                node.cseq = sequence
                if self.is_vcf:
                    node.sequence = self.dict_sequence(node)
                else:
                    node.sequence = self.expanded_sequence(node)
                node.mutations = self.get_mutations(node)

            node.profile = seq2prof(node.cseq, self.gtr.profile_map)
            del node.state # no need to store Fitch states
        self.logger("Done ancestral state reconstruction",3)
        for node in self.tree.get_terminals():
            node.profile = seq2prof(node.original_cseq, self.gtr.profile_map)
        return N_diff

Reconstruct ancestral states using Fitch s algorithm . The method requires sequences to be assigned to leaves . It implements the iteration from leaves to the root constructing the Fitch profiles for each character of the sequence and then by propagating from the root to the leaves reconstructs the sequences of the internal nodes .


def _fitch_state(self, node, pos):
        """
        Determine the Fitch profile for a single character of the node's sequence.
        The profile is essentially the intersection between the children's
        profiles or, if the former is empty, the union of the profiles.

        Parameters
        ----------

         node : PhyloTree.Clade:
            Internal node which the profiles are to be determined

         pos : int
            Position in the node's sequence which the profiles should
            be determinedf for.

        Returns
        -------
         state : numpy.array
            Fitch profile for the character at position pos of the given node.
        """
        state = self._fitch_intersect([k.state[pos] for k in node.clades])
        if len(state) == 0:
            state = np.concatenate([k.state[pos] for k in node.clades])
        return state

def _fitch_state ( self , node , pos ) : state = self . _fitch_intersect ( [ k . state [ pos ] for k in node . clades ] ) if len ( state ) == 0 : state = np . concatenate ( [ k . state [ pos ] for k in node . clades ] ) return state

Determine the Fitch profile for a single character of the node s sequence . The profile is essentially the intersection between the children s profiles or if the former is empty the union of the profiles .


def _fitch_intersect(self, arrays):
        """
        Find the intersection of any number of 1D arrays.
        Return the sorted, unique values that are in all of the input arrays.
        Adapted from numpy.lib.arraysetops.intersect1d
        """
        def pairwise_intersect(arr1, arr2):
            s2 = set(arr2)
            b3 = [val for val in arr1 if val in s2]
            return b3

        arrays = list(arrays) # allow assignment
        N = len(arrays)
        while N > 1:
            arr1 = arrays.pop()
            arr2 = arrays.pop()
            arr = pairwise_intersect(arr1, arr2)
            arrays.append(arr)
            N = len(arrays)

        return arrays[0]

def
_fitch_intersect
(
self
,
arrays
)
:
def
pairwise_intersect
(
arr1
,
arr2
)
:
s2
=
set
(
arr2
)
b3
=
[
val
for
val
in
arr1
if
val
in
s2
]
return
b3
arrays
=
list
(
arrays
)
# allow assignment
N
=
len
(
arrays
)
while
N
>
1
:
arr1
=
arrays
.
pop
(
)
arr2
=
arrays
.
pop
(
)
arr
=
pairwise_intersect
(
arr1
,
arr2
)
arrays
.
append
(
arr
)
N
=
len
(
arrays
)
return
arrays
[
0
]

Find the intersection of any number of 1D arrays . Return the sorted unique values that are in all of the input arrays . Adapted from numpy . lib . arraysetops . intersect1d


def sequence_LH(self, pos=None, full_sequence=False):
        """return the likelihood of the observed sequences given the tree

        Parameters
        ----------
        pos : int, optional
            position in the sequence, if none, the sum over all positions will be returned
        full_sequence : bool, optional
            does the position refer to the full or compressed sequence, by default compressed sequence is assumed.

        Returns
        -------
        float
            likelihood
        """
        if not hasattr(self.tree, "total_sequence_LH"):
            self.logger("TreeAnc.sequence_LH: you need to run marginal ancestral inference first!", 1)
            self.infer_ancestral_sequences(marginal=True)
        if pos is not None:
            if full_sequence:
                compressed_pos = self.full_to_reduced_sequence_map[pos]
            else:
                compressed_pos = pos
            return self.tree.sequence_LH[compressed_pos]
        else:
            return self.tree.total_sequence_LH


def ancestral_likelihood(self):
        """
        Calculate the likelihood of the given realization of the sequences in
        the tree

        Returns
        -------

         log_lh : float
            The tree likelihood given the sequences
        """
        log_lh = np.zeros(self.multiplicity.shape[0])
        for node in self.tree.find_clades(order='postorder'):

            if node.up is None: #  root node
                # 0-1 profile
                profile = seq2prof(node.cseq, self.gtr.profile_map)
                # get the probabilities to observe each nucleotide
                profile *= self.gtr.Pi
                profile = profile.sum(axis=1)
                log_lh += np.log(profile) # product over all characters
                continue

            t = node.branch_length

            indices = np.array([(np.argmax(self.gtr.alphabet==a),
                        np.argmax(self.gtr.alphabet==b)) for a, b in zip(node.up.cseq, node.cseq)])

            logQt = np.log(self.gtr.expQt(t))
            lh = logQt[indices[:, 1], indices[:, 0]]
            log_lh += lh

        return log_lh


def _branch_length_to_gtr(self, node):
        """
        Set branch lengths to either mutation lengths of given branch lengths.
        The assigend values are to be used in the following ML analysis.
        """
        if self.use_mutation_length:
            return max(ttconf.MIN_BRANCH_LENGTH*self.one_mutation, node.mutation_length)
        else:
            return max(ttconf.MIN_BRANCH_LENGTH*self.one_mutation, node.branch_length)

def _branch_length_to_gtr ( self , node ) : if self . use_mutation_length : return max ( ttconf . MIN_BRANCH_LENGTH * self . one_mutation , node . mutation_length ) else : return max ( ttconf . MIN_BRANCH_LENGTH * self . one_mutation , node . branch_length )

Set branch lengths to either mutation lengths of given branch lengths . The assigend values are to be used in the following ML analysis .


def optimize_branch_length(self, mode='joint', **kwargs):
        """
        Perform optimization for the branch lengths of the entire tree.
        This method only does a single path and needs to be iterated.

        **Note** this method assumes that each node stores information
        about its sequence as numpy.array object (node.sequence attribute).
        Therefore, before calling this method, sequence reconstruction with
        either of the available models must be performed.

        Parameters
        ----------

         mode : str
            Optimize branch length assuming the joint ML sequence assignment
            of both ends of the branch (:code:`joint`), or trace over all possible sequence
            assignments on both ends of the branch (:code:`marginal`) (slower, experimental).

         **kwargs :
            Keyword arguments

        Keyword Args
        ------------

         verbose : int
            Output level

         store_old : bool
            If True, the old lengths will be saved in :code:`node._old_dist` attribute.
            Useful for testing, and special post-processing.

        """

        self.logger("TreeAnc.optimize_branch_length: running branch length optimization in mode %s..."%mode,1)
        if (self.tree is None) or (self.aln is None):
            self.logger("TreeAnc.optimize_branch_length: ERROR, alignment or tree are missing", 0)
            return ttconf.ERROR

        store_old_dist = False

        if 'store_old' in kwargs:
            store_old_dist = kwargs['store_old']

        if mode=='marginal':
            # a marginal ancestral reconstruction is required for
            # marginal branch length inference
            if not hasattr(self.tree.root, "marginal_profile"):
                self.infer_ancestral_sequences(marginal=True)

        max_bl = 0
        for node in self.tree.find_clades(order='postorder'):
            if node.up is None: continue # this is the root
            if store_old_dist:
                node._old_length = node.branch_length

            if mode=='marginal':
                new_len = self.optimal_marginal_branch_length(node)
            elif mode=='joint':
                new_len = self.optimal_branch_length(node)
            else:
                self.logger("treeanc.optimize_branch_length: unsupported optimization mode",4, warn=True)
                new_len = node.branch_length

            if new_len < 0:
                continue

            self.logger("Optimization results: old_len=%.4e, new_len=%.4e, naive=%.4e"
                   " Updating branch length..."%(node.branch_length, new_len, len(node.mutations)*self.one_mutation), 5)

            node.branch_length = new_len
            node.mutation_length=new_len
            max_bl = max(max_bl, new_len)

        # as branch lengths changed, the params must be fixed
        self.tree.root.up = None
        self.tree.root.dist2root = 0.0
        if max_bl>0.15 and mode=='joint':
            self.logger("TreeAnc.optimize_branch_length: THIS TREE HAS LONG BRANCHES."
                        " \n\t ****TreeTime IS NOT DESIGNED TO OPTIMIZE LONG BRANCHES."
                        " \n\t ****PLEASE OPTIMIZE BRANCHES WITH ANOTHER TOOL AND RERUN WITH"
                        " \n\t ****branch_length_mode='input'", 0, warn=True)
        self._prepare_nodes()
        return ttconf.SUCCESS

Perform optimization for the branch lengths of the entire tree . This method only does a single path and needs to be iterated .


def optimize_branch_length_global(self, **kwargs):
        """
        EXPERIMENTAL GLOBAL OPTIMIZATION
        """

        self.logger("TreeAnc.optimize_branch_length_global: running branch length optimization...",1)

        def neg_log(s):
            for si, n in zip(s, self.tree.find_clades(order='preorder')):
                n.branch_length = si**2

            self.infer_ancestral_sequences(marginal=True)

            gradient = []
            for si, n in zip(s, self.tree.find_clades(order='preorder')):
                if n.up:
                    pp, pc = self.marginal_branch_profile(n)
                    Qtds = self.gtr.expQsds(si).T
                    Qt = self.gtr.expQs(si).T

                    res = pp.dot(Qt)
                    overlap = np.sum(res*pc, axis=1)

                    res_ds = pp.dot(Qtds)
                    overlap_ds = np.sum(res_ds*pc, axis=1)
                    logP = np.sum(self.multiplicity*overlap_ds/overlap)

                    gradient.append(logP)
                else:
                    gradient.append(2*(si**2-0.001))

            print(-self.tree.sequence_marginal_LH)
            return (-self.tree.sequence_marginal_LH + (s[0]**2-0.001)**2, -1.0*np.array(gradient))

        from scipy.optimize import minimize
        x0 = np.sqrt([n.branch_length for n in self.tree.find_clades(order='preorder')])
        sol = minimize(neg_log, x0, jac=True)

        for new_len, node in zip(sol['x'], self.tree.find_clades()):
            self.logger("Optimization results: old_len=%.4f, new_len=%.4f "
                   " Updating branch length..."%(node.branch_length, new_len), 5)

            node.branch_length = new_len**2
            node.mutation_length=new_len**2

        # as branch lengths changed, the params must be fixed
        self.tree.root.up = None
        self.tree.root.dist2root = 0.0
        self._prepare_nodes()


def optimal_branch_length(self, node):
        '''
        Calculate optimal branch length given the sequences of node and parent

        Parameters
        ----------
        node : PhyloTree.Clade
           TreeNode, attached to the branch.

        Returns
        -------
        new_len : float
           Optimal length of the given branch

        '''
        if node.up is None:
            return self.one_mutation

        parent = node.up
        if hasattr(node, 'compressed_sequence'):
            new_len = self.gtr.optimal_t_compressed(node.compressed_sequence['pair'],
                                                    node.compressed_sequence['multiplicity'])
        else:
            new_len = self.gtr.optimal_t(parent.cseq, node.cseq,
                                         pattern_multiplicity=self.multiplicity,
                                         ignore_gaps=self.ignore_gaps)
        return new_len

def optimal_branch_length ( self , node ) : if node . up is None : return self . one_mutation parent = node . up if hasattr ( node , 'compressed_sequence' ) : new_len = self . gtr . optimal_t_compressed ( node . compressed_sequence [ 'pair' ] , node . compressed_sequence [ 'multiplicity' ] ) else : new_len = self . gtr . optimal_t ( parent . cseq , node . cseq , pattern_multiplicity = self . multiplicity , ignore_gaps = self . ignore_gaps ) return new_len


def optimize_seq_and_branch_len(self,reuse_branch_len=True, prune_short=True,
                                    marginal_sequences=False, branch_length_mode='joint',
                                    max_iter=5, infer_gtr=False, **kwargs):
        """
        Iteratively set branch lengths and reconstruct ancestral sequences until
        the values of either former or latter do not change. The algorithm assumes
        knowing only the topology of the tree, and requires that sequences are assigned
        to all leaves of the tree.

        The first step is to pre-reconstruct ancestral
        states using Fitch reconstruction algorithm or ML using existing branch length
        estimates. Then, optimize branch lengths and re-do reconstruction until
        convergence using ML method.

        Parameters
        -----------

         reuse_branch_len : bool
            If True, rely on the initial branch lengths, and start with the
            maximum-likelihood ancestral sequence inference using existing branch
            lengths. Otherwise, do initial reconstruction of ancestral states with
            Fitch algorithm, which uses only the tree topology.

         prune_short : bool
            If True, the branches with zero optimal length will be pruned from
            the tree, creating polytomies. The polytomies could be further
            processed using :py:meth:`treetime.TreeTime.resolve_polytomies` from the TreeTime class.

         marginal_sequences : bool
            Assign sequences to their marginally most likely value, rather than
            the values that are jointly most likely across all nodes.

         branch_length_mode : str
            'joint', 'marginal', or 'input'. Branch lengths are left unchanged in case
            of 'input'. 'joint' and 'marginal' cause branch length optimization
            while setting sequences to the ML value or tracing over all possible
            internal sequence states.

         max_iter : int
            Maximal number of times sequence and branch length iteration are optimized

         infer_gtr : bool
            Infer a GTR model from the observed substitutions.

        """
        if branch_length_mode=='marginal':
            marginal_sequences = True

        self.logger("TreeAnc.optimize_sequences_and_branch_length: sequences...", 1)
        if reuse_branch_len:
            N_diff = self.reconstruct_anc(method='probabilistic', infer_gtr=infer_gtr,
                                          marginal=marginal_sequences, **kwargs)
            self.optimize_branch_len(verbose=0, store_old=False, mode=branch_length_mode)
        else:
            N_diff = self.reconstruct_anc(method='fitch', infer_gtr=infer_gtr, **kwargs)

            self.optimize_branch_len(verbose=0, store_old=False, marginal=False)

        n = 0
        while n<max_iter:
            n += 1
            if prune_short:
                self.prune_short_branches()
            N_diff = self.reconstruct_anc(method='probabilistic', infer_gtr=False,
                                          marginal=marginal_sequences, **kwargs)

            self.logger("TreeAnc.optimize_sequences_and_branch_length: Iteration %d."
                   " #Nuc changed since prev reconstructions: %d" %(n, N_diff), 2)

            if N_diff < 1:
                break
            self.optimize_branch_len(verbose=0, store_old=False, mode=branch_length_mode)

        self.tree.unconstrained_sequence_LH = (self.tree.sequence_LH*self.multiplicity).sum()
        self._prepare_nodes() # fix dist2root and up-links after reconstruction
        self.logger("TreeAnc.optimize_sequences_and_branch_length: Unconstrained sequence LH:%f" % self.tree.unconstrained_sequence_LH , 2)
        return ttconf.SUCCESS

Iteratively set branch lengths and reconstruct ancestral sequences until the values of either former or latter do not change . The algorithm assumes knowing only the topology of the tree and requires that sequences are assigned to all leaves of the tree .


def get_reconstructed_alignment(self):
        """
        Get the multiple sequence alignment, including reconstructed sequences for
        the internal nodes.

        Returns
        -------
        new_aln : MultipleSeqAlignment
           Alignment including sequences of all internal nodes

        """
        from Bio.Align import MultipleSeqAlignment
        from Bio.Seq import Seq
        from Bio.SeqRecord import SeqRecord
        self.logger("TreeAnc.get_reconstructed_alignment ...",2)
        if not hasattr(self.tree.root, 'sequence'):
            self.logger("TreeAnc.reconstructed_alignment... reconstruction not yet done",3)
            self.reconstruct_anc('probabilistic')

        new_aln = MultipleSeqAlignment([SeqRecord(id=n.name, seq=Seq("".join(n.sequence)), description="")
                                        for n in self.tree.find_clades()])

        return new_aln

Get the multiple sequence alignment including reconstructed sequences for the internal nodes .


def Q(self):
        """function that return the product of the transition matrix
           and the equilibrium frequencies to obtain the rate matrix
           of the GTR model
        """
        tmp = np.einsum('ia,ij->ija', self.Pi, self.W)
        diag_vals = np.sum(tmp, axis=0)
        for x in range(tmp.shape[-1]):
            np.fill_diagonal(tmp[:,:,x], -diag_vals[:,x])
        return tmp

def Q ( self ) : tmp = np . einsum ( 'ia,ij->ija' , self . Pi , self . W ) diag_vals = np . sum ( tmp , axis = 0 ) for x in range ( tmp . shape [ - 1 ] ) : np . fill_diagonal ( tmp [ : , : , x ] , - diag_vals [ : , x ] ) return tmp


def custom(cls, mu=1.0, pi=None, W=None, **kwargs):
        """
        Create a GTR model by specifying the matrix explicitly

        Parameters
        ----------

         mu : float
            Substitution rate

         W : nxn matrix
            Substitution matrix

         pi : n vector
            Equilibrium frequencies

         **kwargs:
            Key word arguments to be passed

        Keyword Args
        ------------

         alphabet : str
            Specify alphabet when applicable. If the alphabet specification is
            required, but no alphabet is specified, the nucleotide alphabet will be used as
            default.

        """
        gtr = cls(**kwargs)
        gtr.assign_rates(mu=mu, pi=pi, W=W)
        return gtr

def custom ( cls , mu = 1.0 , pi = None , W = None , * * kwargs ) : gtr = cls ( * * kwargs ) gtr . assign_rates ( mu = mu , pi = pi , W = W ) return gtr


def standard(model, **kwargs):
        """
        Create standard model of molecular evolution.

        Parameters
        ----------

         model : str
            Model to create. See list of available models below
         **kwargs:
            Key word arguments to be passed to the model

        **Available models**

        - JC69:

          Jukes-Cantor 1969 model. This model assumes equal frequencies
          of the nucleotides and equal transition rates between nucleotide states.
          For more info, see: Jukes and Cantor (1969).
          Evolution of Protein Molecules. New York: Academic Press. pp. 21-132.
          To create this model, use:

          :code:`mygtr = GTR.standard(model='jc69', mu=<my_mu>, alphabet=<my_alph>)`

          :code:`my_mu` - substitution rate (float)

          :code:`my_alph` - alphabet (str: :code:`'nuc'` or  :code:`'nuc_nogap'`)

        - K80:

          Kimura 1980 model. Assumes equal concentrations across nucleotides, but
          allows different rates between transitions and transversions. The ratio
          of the transversion/transition rates is given by kappa parameter.
          For more info, see
          Kimura (1980),  J. Mol. Evol. 16 (2): 111-120. doi:10.1007/BF01731581.
          Current implementation of the model does not account for the gaps.

          :code:`mygtr = GTR.standard(model='k80', mu=<my_mu>, kappa=<my_kappa>)`

          :code:`mu` - overall substitution rate (float)

          :code:`kappa` - ratio of transversion/transition rates (float)

        - F81:

          Felsenstein 1981 model. Assumes non-equal concentrations across nucleotides,
          but the transition rate between all states is assumed to be equal. See
          Felsenstein (1981), J. Mol. Evol. 17  (6): 368-376. doi:10.1007/BF01734359
          for details.

          :code:`mygtr = GTR.standard(model='F81', mu=<mu>, pi=<pi>, alphabet=<alph>)`

          :code:`mu` -  substitution rate  (float)

          :code:`pi`  - : nucleotide concentrations (numpy.array)

          :code:`alphabet' -  alphabet to use. (:code:`'nuc'` or  :code:`'nuc_nogap'`)

        - HKY85:

          Hasegawa, Kishino and Yano 1985 model. Allows different concentrations of the
          nucleotides (as in F81) + distinguishes between transition/transversion substitutions
          (similar to K80). Link:
          Hasegawa, Kishino, Yano (1985), J. Mol. Evol. 22 (2): 160-174. doi:10.1007/BF02101694

          Current implementation of the model does not account for the gaps

          :code:`mygtr = GTR.standard(model='HKY85', mu=<mu>, pi=<pi>, kappa=<kappa>)`

          :code:`mu` -  substitution rate  (float)

          :code:`pi`  - : nucleotide concentrations (numpy.array)

          :code:`kappa` - ratio of transversion/transition rates (float)

        - T92:

          Tamura 1992 model. Extending Kimura  (1980) model for the case where a
          G+C-content bias exists. Link:
          Tamura K (1992),  Mol.  Biol. Evol. 9 (4): 678-687.  DOI: 10.1093/oxfordjournals.molbev.a040752

          Current implementation of the model does not account for the gaps

          :code:`mygtr = GTR.standard(model='T92', mu=<mu>, pi_GC=<pi_gc>, kappa=<kappa>)`

          :code:`mu` -  substitution rate  (float)

          :code:`pi_GC`  - : relative GC content

          :code:`kappa` - ratio of transversion/transition rates (float)

        - TN93:

          Tamura and Nei 1993. The model distinguishes between the two different types of
          transition: (A <-> G) is allowed to have a different rate to (C<->T).
          Transversions have the same rate. The frequencies of the nucleotides are allowed
          to be different. Link: Tamura, Nei (1993), MolBiol Evol. 10 (3): 512-526.
          DOI:10.1093/oxfordjournals.molbev.a040023

          :code:`mygtr = GTR.standard(model='TN93', mu=<mu>, kappa1=<k1>, kappa2=<k2>)`

          :code:`mu` -  substitution rate  (float)

          :code:`kappa1`  - relative A<-->C, A<-->T, T<-->G and G<-->C rates (float)

          :code:`kappa` - relative C<-->T rate (float)

          .. Note::
                Rate of A<-->G substitution is set to one. All other rates
                (kappa1, kappa2) are specified relative to this rate

        """
        from .nuc_models import JC69, K80, F81, HKY85, T92, TN93
        from .aa_models  import JTT92

        if model.lower() in ['jc', 'jc69', 'jukes-cantor', 'jukes-cantor69', 'jukescantor', 'jukescantor69']:
            return JC69(**kwargs)
        elif model.lower() in ['k80', 'kimura80', 'kimura1980']:
            return K80(**kwargs)
        elif model.lower() in ['f81', 'felsenstein81', 'felsenstein1981']:
            return F81(**kwargs)
        elif model.lower() in ['hky', 'hky85', 'hky1985']:
            return HKY85(**kwargs)
        elif model.lower() in ['t92', 'tamura92', 'tamura1992']:
            return T92(**kwargs)
        elif model.lower() in ['tn93', 'tamura_nei_93', 'tamuranei93']:
            return TN93(**kwargs)
        elif model.lower() in ['jtt', 'jtt92']:
            return JTT92(**kwargs)
        else:
            raise KeyError("The GTR model '{}' is not in the list of available models."
                "".format(model))

Create standard model of molecular evolution .


def _check_fix_Q(self, fixed_mu=False):
        """
        Check the main diagonal of Q and fix it in case it does not corresond
        the definition of the rate matrix. Should be run every time when creating
        custom GTR model.
        """
        # fix Q
        self.Pi /= self.Pi.sum() # correct the Pi manually
        # NEEDED TO BREAK RATE MATRIX DEGENERACY AND FORCE NP TO RETURN REAL ORTHONORMAL EIGENVECTORS
        self.W += self.break_degen + self.break_degen.T
        # fix W
        np.fill_diagonal(self.W, 0)
        Wdiag = -(self.Q).sum(axis=0)/self.Pi
        np.fill_diagonal(self.W, Wdiag)
        scale_factor = -np.sum(np.diagonal(self.Q)*self.Pi)
        self.W /= scale_factor
        if not fixed_mu:
            self.mu *= scale_factor
        if (self.Q.sum(axis=0) < 1e-10).sum() <  self.alphabet.shape[0]: # fix failed
            print ("Cannot fix the diagonal of the GTR rate matrix. Should be all zero", self.Q.sum(axis=0))
            import ipdb; ipdb.set_trace()
            raise ArithmeticError("Cannot fix the diagonal of the GTR rate matrix.")

Check the main diagonal of Q and fix it in case it does not corresond the definition of the rate matrix . Should be run every time when creating custom GTR model .


def prob_t_compressed(self, seq_pair, multiplicity, t, return_log=False):
        '''
        Calculate the probability of observing a sequence pair at a distance t,
        for compressed sequences

        Parameters
        ----------

          seq_pair : numpy array
            :code:`np.array([(0,1), (2,2), ()..])` as indicies of
            pairs of aligned positions. (e.g. 'A'==0, 'C'==1 etc).
            This only lists all occuring parent-child state pairs, order is irrelevant

          multiplicity : numpy array
            The number of times a parent-child state pair is observed.
            This allows compression of the sequence representation

          t : float
            Length of the branch separating parent and child

          return_log : bool
            Whether or not to exponentiate the result

        '''
        if t<0:
            logP = -ttconf.BIG_NUMBER
        else:
            tmp_eQT = self.expQt(t)
            bad_indices=(tmp_eQT==0)
            logQt = np.log(tmp_eQT + ttconf.TINY_NUMBER*(bad_indices))
            logQt[np.isnan(logQt) | np.isinf(logQt) | bad_indices] = -ttconf.BIG_NUMBER
            logP = np.sum(logQt[seq_pair[:,1], seq_pair[:,0]]*multiplicity)

        return logP if return_log else np.exp(logP)

def prob_t_compressed ( self , seq_pair , multiplicity , t , return_log = False ) : if t < 0 : logP = - ttconf . BIG_NUMBER else : tmp_eQT = self . expQt ( t ) bad_indices = ( tmp_eQT == 0 ) logQt = np . log ( tmp_eQT + ttconf . TINY_NUMBER * ( bad_indices ) ) logQt [ np . isnan ( logQt ) | np . isinf ( logQt ) | bad_indices ] = - ttconf . BIG_NUMBER logP = np . sum ( logQt [ seq_pair [ : , 1 ] , seq_pair [ : , 0 ] ] * multiplicity ) return logP if return_log else np . exp ( logP )

Calculate the probability of observing a sequence pair at a distance t for compressed sequences


def optimal_t(self, seq_p, seq_ch, pattern_multiplicity=None, ignore_gaps=False):
        '''
        Find the optimal distance between the two sequences

        Parameters
        ----------

         seq_p : character array
            Parent sequence

         seq_c : character array
            Child sequence

         pattern_multiplicity : numpy array
            If sequences are reduced by combining identical alignment patterns,
            these multplicities need to be accounted for when counting the number
            of mutations across a branch. If None, all pattern are assumed to
            occur exactly once.

         ignore_gaps : bool
            If True, ignore gaps in distance calculations

        '''
        seq_pair, multiplicity = self.compress_sequence_pair(seq_p, seq_ch,
                                                            pattern_multiplicity = pattern_multiplicity,
                                                            ignore_gaps=ignore_gaps)
        return self.optimal_t_compressed(seq_pair, multiplicity)

def optimal_t ( self , seq_p , seq_ch , pattern_multiplicity = None , ignore_gaps = False ) : seq_pair , multiplicity = self . compress_sequence_pair ( seq_p , seq_ch , pattern_multiplicity = pattern_multiplicity , ignore_gaps = ignore_gaps ) return self . optimal_t_compressed ( seq_pair , multiplicity )


def optimal_t_compressed(self, seq_pair, multiplicity, profiles=False, tol=1e-10):
        """
        Find the optimal distance between the two sequences, for compressed sequences

        Parameters
        ----------

         seq_pair : compressed_sequence_pair
            Compressed representation of sequences along a branch, either
            as tuple of state pairs or as tuple of profiles.

         multiplicity : array
            Number of times each state pair in seq_pair appears (if profile==False)

            Number of times an alignment pattern is observed (if profiles==True)

         profiles : bool, default False
            The standard branch length optimization assumes fixed sequences at
            either end of the branch. With profiles==True, optimization is performed
            while summing over all possible states of the nodes at either end of the
            branch. Note that the meaning/format of seq_pair and multiplicity
            depend on the value of profiles.

        """

        def _neg_prob(t, seq_pair, multiplicity):
            """
            Probability to observe a child given the the parent state, transition
            matrix, and the time of evolution (branch length).

            Parameters
            ----------

             t : double
                Branch length (time between sequences)

             parent :  numpy.array
                Parent sequence

             child : numpy.array
                Child sequence

             tm :  GTR
                Model of evolution

            Returns
            -------

             prob : double
                Negative probability of the two given sequences
                to be separated by the time t.
            """
            if profiles:
                res = -1.0*self.prob_t_profiles(seq_pair, multiplicity,t**2, return_log=True)
                return res
            else:
                return -1.0*self.prob_t_compressed(seq_pair, multiplicity,t**2, return_log=True)

        try:
            from scipy.optimize import minimize_scalar
            opt = minimize_scalar(_neg_prob,
                    bounds=[-np.sqrt(ttconf.MAX_BRANCH_LENGTH),np.sqrt(ttconf.MAX_BRANCH_LENGTH)],
                    args=(seq_pair, multiplicity), tol=tol)
            new_len = opt["x"]**2
            if 'success' not in opt:
                opt['success'] = True
                self.logger("WARNING: the optimization result does not contain a 'success' flag:"+str(opt),4, warn=True)
        except:
            import scipy
            print('legacy scipy', scipy.__version__)
            from scipy.optimize import fminbound
            new_len = fminbound(_neg_prob,
                      -np.sqrt(ttconf.MAX_BRANCH_LENGTH),np.sqrt(ttconf.MAX_BRANCH_LENGTH),
                       args=(seq_pair, multiplicity))
            new_len = new_len**2
            opt={'success':True}

        if new_len > .9 * ttconf.MAX_BRANCH_LENGTH:
            self.logger("WARNING: GTR.optimal_t_compressed -- The branch length seems to be very long!", 4, warn=True)

        if opt["success"] != True:
            # return hamming distance: number of state pairs where state differs/all pairs
            new_len =  np.sum(multiplicity[seq_pair[:,1]!=seq_pair[:,0]])/np.sum(multiplicity)

        return new_len

Find the optimal distance between the two sequences for compressed sequences


def prob_t_profiles(self, profile_pair, multiplicity, t,
                        return_log=False, ignore_gaps=True):
        '''
        Calculate the probability of observing a node pair at a distance t

        Parameters
        ----------

          profile_pair: numpy arrays
            Probability distributions of the nucleotides at either
            end of the branch. pp[0] = parent, pp[1] = child

          multiplicity : numpy array
            The number of times an alignment pattern is observed

          t : float
            Length of the branch separating parent and child

          ignore_gaps: bool
            If True, ignore mutations to and from gaps in distance calculations

          return_log : bool
            Whether or not to exponentiate the result

        '''
        if t<0:
            logP = -ttconf.BIG_NUMBER
        else:
            Qt = self.expQt(t)
            if len(Qt.shape)==3:
                res = np.einsum('ai,ija,aj->a', profile_pair[1], Qt, profile_pair[0])
            else:
                res = np.einsum('ai,ij,aj->a', profile_pair[1], Qt, profile_pair[0])
            if ignore_gaps and (self.gap_index is not None): # calculate the probability that neither outgroup/node has a gap
                non_gap_frac = (1-profile_pair[0][:,self.gap_index])*(1-profile_pair[1][:,self.gap_index])
                # weigh log LH by the non-gap probability
                logP = np.sum(multiplicity*np.log(res)*non_gap_frac)
            else:
                logP = np.sum(multiplicity*np.log(res))

        return logP if return_log else np.exp(logP)


def evolve(self, profile, t, return_log=False):
        """
        Compute the probability of the sequence state of the child
        at time t later, given the parent profile.

        Parameters
        ----------

         profile : numpy.array
            Sequence profile. Shape = (L, a),
            where L - sequence length, a - alphabet size.

         t : double
            Time to propagate

         return_log: bool
            If True, return log-probability

        Returns
        -------

         res : np.array
            Profile of the sequence after time t in the future.
            Shape = (L, a), where L - sequence length, a - alphabet size.

        """
        Qt = self.expQt(t).T
        res = profile.dot(Qt)
        return np.log(res) if return_log else res

def evolve ( self , profile , t , return_log = False ) : Qt = self . expQt ( t ) . T res = profile . dot ( Qt ) return np . log ( res ) if return_log else res

Compute the probability of the sequence state of the child at time t later given the parent profile .


def sequence_logLH(self,seq, pattern_multiplicity=None):
        """
        Returns the log-likelihood of sampling a sequence from equilibrium frequency.
        Expects a sequence as numpy array

        Parameters
        ----------

         seq : numpy array
            Compressed sequence as an array of chars

         pattern_multiplicity : numpy_array
            The number of times each position in sequence is observed in the
            initial alignment. If None, sequence is assumed to be not compressed

        """
        if pattern_multiplicity is None:
            pattern_multiplicity = np.ones_like(seq, dtype=float)
        return np.sum([np.sum((seq==state)*pattern_multiplicity*np.log(self.Pi[si]))
                      for si,state in enumerate(self.alphabet)])

def sequence_logLH ( self , seq , pattern_multiplicity = None ) : if pattern_multiplicity is None : pattern_multiplicity = np . ones_like ( seq , dtype = float ) return np . sum ( [ np . sum ( ( seq == state ) * pattern_multiplicity * np . log ( self . Pi [ si ] ) ) for si , state in enumerate ( self . alphabet ) ] )

Returns the log - likelihood of sampling a sequence from equilibrium frequency . Expects a sequence as numpy array


def _set_branch_length_mode(self, branch_length_mode):
        '''
        if branch_length mode is not explicitly set, set according to
        empirical branch length distribution in input tree

        Parameters
        ----------

         branch_length_mode : str, 'input', 'joint', 'marginal'
            if the maximal branch length in the tree is longer than 0.05, this will
            default to 'input'. Otherwise set to 'joint'
        '''
        if branch_length_mode in ['joint', 'marginal', 'input']:
            self.branch_length_mode = branch_length_mode
        elif self.aln:
            bl_dis = [n.branch_length for n in self.tree.find_clades() if n.up]
            max_bl = np.max(bl_dis)
            if max_bl>0.1:
                bl_mode = 'input'
            else:
                bl_mode = 'joint'
            self.logger("TreeTime._set_branch_length_mode: maximum branch length is %1.3e, using branch length mode %s"%(max_bl, bl_mode),1)
            self.branch_length_mode = bl_mode
        else:
            self.branch_length_mode = 'input'

if branch_length mode is not explicitly set set according to empirical branch length distribution in input tree


def clock_filter(self, reroot='least-squares', n_iqd=None, plot=False):
        '''
        Labels outlier branches that don't seem to follow a molecular clock
        and excludes them from subsequent molecular clock estimation and
        the timetree propagation.

        Parameters
        ----------
         reroot : str
            Method to find the best root in the tree (see :py:meth:`treetime.TreeTime.reroot` for options)

         n_iqd : int
            Number of iqd intervals. The outlier nodes are those which do not fall
            into :math:`IQD\cdot n_iqd` interval (:math:`IQD` is the interval between
            75\ :sup:`th` and 25\ :sup:`th` percentiles)

            If None, the default (3) assumed

         plot : bool
            If True, plot the results

        '''
        if n_iqd is None:
            n_iqd = ttconf.NIQD
        if type(reroot) is list and len(reroot)==1:
            reroot=str(reroot[0])

        terminals = self.tree.get_terminals()
        if reroot:
            if self.reroot(root='least-squares' if reroot=='best' else reroot, covariation=False)==ttconf.ERROR:
                return ttconf.ERROR
        else:
            self.get_clock_model(covariation=False)

        clock_rate = self.clock_model['slope']
        icpt = self.clock_model['intercept']
        res = {}
        for node in terminals:
            if hasattr(node, 'raw_date_constraint') and  (node.raw_date_constraint is not None):
                res[node] = node.dist2root - clock_rate*np.mean(node.raw_date_constraint) - icpt

        residuals = np.array(list(res.values()))
        iqd = np.percentile(residuals,75) - np.percentile(residuals,25)
        for node,r in res.items():
            if abs(r)>n_iqd*iqd and node.up.up is not None:
                self.logger('TreeTime.ClockFilter: marking %s as outlier, residual %f interquartile distances'%(node.name,r/iqd), 3, warn=True)
                node.bad_branch=True
            else:
                node.bad_branch=False

        # redo root estimation after outlier removal
        if reroot and self.reroot(root=reroot)==ttconf.ERROR:
                return ttconf.ERROR

        if plot:
            self.plot_root_to_tip()

        return ttconf.SUCCESS

Labels outlier branches that don t seem to follow a molecular clock and excludes them from subsequent molecular clock estimation and the timetree propagation .


def plot_root_to_tip(self, add_internal=False, label=True, ax=None):
        """
        Plot root-to-tip regression

        Parameters
        ----------
        add_internal : bool
           If true, plot inte`rnal node positions

        label : bool
           If true, label the plots

        ax : matplotlib axes
           If not None, use the provided matplotlib axes to plot the results
        """
        Treg = self.setup_TreeRegression()
        if self.clock_model and 'cov' in self.clock_model:
            cf = self.clock_model['valid_confidence']
        else:
            cf = False
        Treg.clock_plot(ax=ax, add_internal=add_internal, confidence=cf, n_sigma=2,
                        regression=self.clock_model)

def plot_root_to_tip ( self , add_internal = False , label = True , ax = None ) : Treg = self . setup_TreeRegression ( ) if self . clock_model and 'cov' in self . clock_model : cf = self . clock_model [ 'valid_confidence' ] else : cf = False Treg . clock_plot ( ax = ax , add_internal = add_internal , confidence = cf , n_sigma = 2 , regression = self . clock_model )


def resolve_polytomies(self, merge_compressed=False):
        """
        Resolve the polytomies on the tree.

        The function scans the tree, resolves polytomies if present,
        and re-optimizes the tree with new topology. Note that polytomies are only
        resolved if that would result in higher likelihood. Sometimes, stretching
        two or more branches that carry several mutations is less costly than
        an additional branch with zero mutations (long branches are not stiff,
        short branches are).

        Parameters
        ----------
         merge_compressed : bool
            If True, keep compressed branches as polytomies. If False,
            return a strictly binary tree.

        Returns
        --------
         poly_found : int
            The number of polytomies found

        """
        self.logger("TreeTime.resolve_polytomies: resolving multiple mergers...",1)
        poly_found=0

        for n in self.tree.find_clades():
            if len(n.clades) > 2:
                prior_n_clades = len(n.clades)
                self._poly(n, merge_compressed)
                poly_found+=prior_n_clades - len(n.clades)

        obsolete_nodes = [n for n in self.tree.find_clades() if len(n.clades)==1 and n.up is not None]
        for node in obsolete_nodes:
            self.logger('TreeTime.resolve_polytomies: remove obsolete node '+node.name,4)
            if node.up is not None:
                self.tree.collapse(node)

        if poly_found:
            self.logger('TreeTime.resolve_polytomies: introduces %d new nodes'%poly_found,3)
        else:
            self.logger('TreeTime.resolve_polytomies: No more polytomies to resolve',3)
        return poly_found


def print_lh(self, joint=True):
        """
        Print the total likelihood of the tree given the constrained leaves

        Parameters
        ----------

         joint : bool
            If true, print joint LH, else print marginal LH

        """
        try:
            u_lh = self.tree.unconstrained_sequence_LH
            if joint:
                s_lh = self.tree.sequence_joint_LH
                t_lh = self.tree.positional_joint_LH
                c_lh = self.tree.coalescent_joint_LH
            else:
                s_lh = self.tree.sequence_marginal_LH
                t_lh = self.tree.positional_marginal_LH
                c_lh = 0

            print ("###  Tree Log-Likelihood  ###\n"
                " Sequence log-LH without constraints: \t%1.3f\n"
                " Sequence log-LH with constraints:    \t%1.3f\n"
                " TreeTime sequence log-LH:            \t%1.3f\n"
                " Coalescent log-LH:                   \t%1.3f\n"
               "#########################"%(u_lh, s_lh,t_lh, c_lh))
        except:
            print("ERROR. Did you run the corresponding inference (joint/marginal)?")

" TreeTime sequence log-LH:            \t%1.3f\n"

" Coalescent log-LH:                   \t%1.3f\n"


def add_coalescent_model(self, Tc, **kwargs):
        """Add a coalescent model to the tree and optionally optimze

        Parameters
        ----------
        Tc : float,str
            If this is a float, it will be interpreted as the inverse merger
            rate in molecular clock units, if its is a
        """
        from .merger_models import Coalescent
        self.logger('TreeTime.run: adding coalescent prior with Tc='+str(Tc),1)
        self.merger_model = Coalescent(self.tree,
                                       date2dist=self.date2dist, logger=self.logger)

        if Tc=='skyline': # restrict skyline model optimization to last iteration
            self.merger_model.optimize_skyline(**kwargs)
            self.logger("optimized a skyline ", 2)
        else:
            if Tc in ['opt', 'const']:
                self.merger_model.optimize_Tc()
                self.logger("optimized Tc to %f"%self.merger_model.Tc.y[0], 2)
            else:
                try:
                    self.merger_model.set_Tc(Tc)
                except:
                    self.logger("setting of coalescent time scale failed", 1, warn=True)

        self.merger_model.attach_to_tree()


def _find_best_root(self, covariation=True, force_positive=True, slope=0, **kwarks):
        '''
        Determine the node that, when the tree is rooted on this node, results
        in the best regression of temporal constraints and root to tip distances.

        Parameters
        ----------

         infer_gtr : bool
            If True, infer new GTR model after re-root

         covariation : bool
            account for covariation structure when rerooting the tree

         force_positive : bool
            only accept positive evolutionary rate estimates when rerooting the tree

        '''
        for n in self.tree.find_clades():
            n.branch_length=n.mutation_length
        self.logger("TreeTime._find_best_root: searching for the best root position...",2)
        Treg = self.setup_TreeRegression(covariation=covariation)
        return Treg.optimal_reroot(force_positive=force_positive, slope=slope)['node']

Determine the node that when the tree is rooted on this node results in the best regression of temporal constraints and root to tip distances .


def assure_tree(params, tmp_dir='treetime_tmp'):
    """
    Function that attempts to load a tree and build it from the alignment
    if no tree is provided.
    """
    if params.tree is None:
        params.tree = os.path.basename(params.aln)+'.nwk'
        print("No tree given: inferring tree")
        utils.tree_inference(params.aln, params.tree, tmp_dir = tmp_dir)

    if os.path.isdir(tmp_dir):
        shutil.rmtree(tmp_dir)

    try:
        tt = TreeAnc(params.tree)
    except:
        print("Tree loading/building failed.")
        return 1
    return 0

def
assure_tree
(
params
,
tmp_dir
=
'treetime_tmp'
)
:
if
params
.
tree
is
:
params
.
tree
=
os
.
path
.
basename
(
params
.
aln
)
+
'.nwk'
print
(
"No tree given: inferring tree"
)
utils
.
tree_inference
(
params
.
aln
,
params
.
tree
,
tmp_dir
=
tmp_dir
)
if
os
.
path
.
isdir
(
tmp_dir
)
:
shutil
.
rmtree
(
tmp_dir
)
try
:
tt
=
TreeAnc
(
params
.
tree
)
except
:
print
(
"Tree loading/building failed."
)
return
1
return
0

Function that attempts to load a tree and build it from the alignment if no tree is provided .


def create_gtr(params):
    """
    parse the arguments referring to the GTR model and return a GTR structure
    """
    model = params.gtr
    gtr_params = params.gtr_params
    if model == 'infer':
        gtr = GTR.standard('jc', alphabet='aa' if params.aa else 'nuc')
    else:
        try:
            kwargs = {}
            if gtr_params is not None:
                for param in gtr_params:
                    keyval = param.split('=')
                    if len(keyval)!=2: continue
                    if keyval[0] in ['pis', 'pi', 'Pi', 'Pis']:
                        keyval[0] = 'pi'
                        keyval[1] = list(map(float, keyval[1].split(',')))
                    elif keyval[0] not in ['alphabet']:
                        keyval[1] = float(keyval[1])
                    kwargs[keyval[0]] = keyval[1]
            else:
                print ("GTR params are not specified. Creating GTR model with default parameters")

            gtr = GTR.standard(model, **kwargs)
            infer_gtr = False
        except:
            print ("Could not create GTR model from input arguments. Using default (Jukes-Cantor 1969)")
            gtr = GTR.standard('jc', alphabet='aa' if params.aa else 'nuc')
            infer_gtr = False
    return gtr


def read_if_vcf(params):
    """
    Checks if input is VCF and reads in appropriately if it is
    """
    ref = None
    aln = params.aln
    fixed_pi = None
    if hasattr(params, 'aln') and params.aln is not None:
        if any([params.aln.lower().endswith(x) for x in ['.vcf', '.vcf.gz']]):
            if not params.vcf_reference:
                print("ERROR: a reference Fasta is required with VCF-format alignments")
                return -1
            compress_seq = read_vcf(params.aln, params.vcf_reference)
            sequences = compress_seq['sequences']
            ref = compress_seq['reference']
            aln = sequences

            if not hasattr(params, 'gtr') or params.gtr=="infer": #if not specified, set it:
                alpha = alphabets['aa'] if params.aa else alphabets['nuc']
                fixed_pi = [ref.count(base)/len(ref) for base in alpha]
                if fixed_pi[-1] == 0:
                    fixed_pi[-1] = 0.05
                    fixed_pi = [v-0.01 for v in fixed_pi]

    return aln, ref, fixed_pi


def ancestral_reconstruction(params):
    """
    implementing treetime ancestral
    """

    # set up
    if assure_tree(params, tmp_dir='ancestral_tmp'):
        return 1

    outdir = get_outdir(params, '_ancestral')
    basename = get_basename(params, outdir)

    gtr = create_gtr(params)

    ###########################################################################
    ### READ IN VCF
    ###########################################################################
    #sets ref and fixed_pi to None if not VCF
    aln, ref, fixed_pi = read_if_vcf(params)
    is_vcf = True if ref is not None else False

    treeanc = TreeAnc(params.tree, aln=aln, ref=ref, gtr=gtr, verbose=1,
                      fill_overhangs=not params.keep_overhangs)
    ndiff =treeanc.infer_ancestral_sequences('ml', infer_gtr=params.gtr=='infer',
                                             marginal=params.marginal, fixed_pi=fixed_pi)
    if ndiff==ttconf.ERROR: # if reconstruction failed, exit
        return 1

    ###########################################################################
    ### OUTPUT and saving of results
    ###########################################################################
    if params.gtr=="infer":
        print('\nInferred GTR model:')
        print(treeanc.gtr)

    export_sequences_and_tree(treeanc, basename, is_vcf, params.zero_based,
                              report_ambiguous=params.report_ambiguous)

    return 0


def calc_fwhm(distribution, is_neg_log=True):
        """
        Assess the width of the probability distribution. This returns
        full-width-half-max
        """

        if isinstance(distribution, interp1d):

            if is_neg_log:
                ymin = distribution.y.min()
                log_prob = distribution.y-ymin
            else:
                log_prob = -np.log(distribution.y)
                log_prob -= log_prob.min()

            xvals = distribution.x

        elif isinstance(distribution, Distribution):
            # Distribution always stores neg log-prob with the peak value subtracted
            xvals = distribution._func.x
            log_prob = distribution._func.y
        else:
            raise TypeError("Error in computing the FWHM for the distribution. "
                " The input should be either Distribution or interpolation object");

        L = xvals.shape[0]
        # 0.69... is log(2), there is always one value for which this is true since
        # the minimum is subtracted
        tmp = np.where(log_prob < 0.693147)[0]
        x_l, x_u = tmp[0], tmp[-1]
        if L < 2:
            print ("Not enough points to compute FWHM: returning zero")
            return min(TINY_NUMBER, distribution.xmax - distribution.xmin)
        else:
            # need to guard against out-of-bounds errors
            return max(TINY_NUMBER, xvals[min(x_u+1,L-1)] - xvals[max(0,x_l-1)])

Assess the width of the probability distribution . This returns full - width - half - max


def delta_function(cls, x_pos, weight=1., min_width=MIN_INTEGRATION_PEAK):
        """
        Create delta function distribution.
        """

        distribution = cls(x_pos,0.,is_log=True, min_width=min_width)
        distribution.weight  = weight
        return distribution

def delta_function ( cls , x_pos , weight = 1. , min_width = MIN_INTEGRATION_PEAK ) : distribution = cls ( x_pos , 0. , is_log = True , min_width = min_width ) distribution . weight = weight return distribution


def multiply(dists):
        '''
        multiplies a list of Distribution objects
        '''
        if  not all([isinstance(k, Distribution) for k in dists]):
            raise NotImplementedError("Can only multiply Distribution objects")

        n_delta = np.sum([k.is_delta for k in dists])
        min_width = np.max([k.min_width for k in dists])
        if n_delta>1:
            raise ArithmeticError("Cannot multiply more than one delta functions!")
        elif n_delta==1:
            delta_dist_ii = np.where([k.is_delta for k in dists])[0][0]
            delta_dist = dists[delta_dist_ii]
            new_xpos = delta_dist.peak_pos
            new_weight  = np.prod([k.prob(new_xpos) for k in dists if k!=delta_dist_ii]) * delta_dist.weight
            res = Distribution.delta_function(new_xpos, weight = new_weight,min_width=min_width)
        else:
            new_xmin = np.max([k.xmin for k in dists])
            new_xmax = np.min([k.xmax for k in dists])

            x_vals = np.unique(np.concatenate([k.x for k in dists]))
            x_vals = x_vals[(x_vals>new_xmin-TINY_NUMBER)&(x_vals<new_xmax+TINY_NUMBER)]
            y_vals = np.sum([k.__call__(x_vals) for k in dists], axis=0)
            peak = y_vals.min()
            ind = (y_vals-peak)<BIG_NUMBER/1000
            n_points = ind.sum()
            if n_points == 0:
                print ("ERROR in distribution multiplication: Distributions do not overlap")
                x_vals = [0,1]
                y_vals = [BIG_NUMBER,BIG_NUMBER]
                res = Distribution(x_vals, y_vals, is_log=True,
                                   min_width=min_width, kind='linear')
            elif n_points == 1:
                res = Distribution.delta_function(x_vals[0])
            else:
                res = Distribution(x_vals[ind], y_vals[ind], is_log=True,
                                   min_width=min_width, kind='linear', assume_sorted=True)

        return res


def _assign_dates(self):
        """assign dates to nodes

        Returns
        -------
        str
            success/error code
        """
        if self.tree is None:
            self.logger("ClockTree._assign_dates: tree is not set, can't assign dates", 0)
            return ttconf.ERROR

        bad_branch_counter = 0
        for node in self.tree.find_clades(order='postorder'):
            if node.name in self.date_dict:
                tmp_date = self.date_dict[node.name]
                if np.isscalar(tmp_date) and np.isnan(tmp_date):
                    self.logger("WARNING: ClockTree.init: node %s has a bad date: %s"%(node.name, str(tmp_date)), 2, warn=True)
                    node.raw_date_constraint = None
                    node.bad_branch = True
                else:
                    try:
                        tmp = np.mean(tmp_date)
                        node.raw_date_constraint = tmp_date
                        node.bad_branch = False
                    except:
                        self.logger("WARNING: ClockTree.init: node %s has a bad date: %s"%(node.name, str(tmp_date)), 2, warn=True)
                        node.raw_date_constraint = None
                        node.bad_branch = True
            else: # nodes without date contraints

                node.raw_date_constraint = None

                if node.is_terminal():
                    # Terminal branches without date constraints marked as 'bad'
                    node.bad_branch = True
                else:
                    # If all branches dowstream are 'bad', and there is no date constraint for
                    # this node, the branch is marked as 'bad'
                    node.bad_branch = np.all([x.bad_branch for x in node])

            if node.is_terminal() and node.bad_branch:
                bad_branch_counter += 1

        if bad_branch_counter>self.tree.count_terminals()-3:
            self.logger("ERROR: ALMOST NO VALID DATE CONSTRAINTS, EXITING", 1, warn=True)
            return ttconf.ERROR

        return ttconf.SUCCESS


def setup_TreeRegression(self, covariation=True):
        """instantiate a TreeRegression object and set its tip_value and branch_value function
        to defaults that are sensible for treetime instances.

        Parameters
        ----------
        covariation : bool, optional
            account for phylogenetic covariation
        Returns
        -------
        TreeRegression
            a TreeRegression instance with self.tree attached as tree.
        """
        from .treeregression import TreeRegression
        tip_value = lambda x:np.mean(x.raw_date_constraint) if (x.is_terminal() and (x.bad_branch is False)) else None
        branch_value = lambda x:x.mutation_length
        if covariation:
            om = self.one_mutation
            branch_variance = lambda x:((x.clock_length if hasattr(x,'clock_length') else x.mutation_length)
                                        +(self.tip_slack**2*om if x.is_terminal() else 0.0))*om
        else:
            branch_variance = lambda x:1.0 if x.is_terminal() else 0.0

        Treg = TreeRegression(self.tree, tip_value=tip_value,
                             branch_value=branch_value, branch_variance=branch_variance)
        Treg.valid_confidence = covariation
        return Treg

def setup_TreeRegression ( self , covariation = True ) : from . treeregression import TreeRegression tip_value = lambda x : np . mean ( x . raw_date_constraint ) if ( x . is_terminal ( ) and ( x . bad_branch is False ) ) else None branch_value = lambda x : x . mutation_length if covariation : om = self . one_mutation branch_variance = lambda x : ( ( x . clock_length if hasattr ( x , 'clock_length' ) else x . mutation_length ) + ( self . tip_slack ** 2 * om if x . is_terminal ( ) else 0.0 ) ) * om else : branch_variance = lambda x : 1.0 if x . is_terminal ( ) else 0.0 Treg = TreeRegression ( self . tree , tip_value = tip_value , branch_value = branch_value , branch_variance = branch_variance ) Treg . valid_confidence = covariation return Treg

instantiate a TreeRegression object and set its tip_value and branch_value function to defaults that are sensible for treetime instances .


def make_time_tree(self, time_marginal=False, clock_rate=None, **kwargs):
        '''
        Use the date constraints to calculate the most likely positions of
        unconstrained nodes.

        Parameters
        ----------

         time_marginal : bool
            If true, use marginal reconstruction for node positions

         **kwargs
            Key word arguments to initialize dates constraints

        '''
        self.logger("ClockTree: Maximum likelihood tree optimization with temporal constraints",1)

        self.init_date_constraints(clock_rate=clock_rate, **kwargs)

        if time_marginal:
            self._ml_t_marginal(assign_dates = time_marginal=="assign")
        else:
            self._ml_t_joint()

        self.convert_dates()

Use the date constraints to calculate the most likely positions of unconstrained nodes .


def timetree_likelihood(self):
        '''
        Return the likelihood of the data given the current branch length in the tree
        '''
        LH = 0
        for node in self.tree.find_clades(order='preorder'):  # sum the likelihood contributions of all branches
            if node.up is None: # root node
                continue
            LH -= node.branch_length_interpolator(node.branch_length)

        # add the root sequence LH and return
        if self.aln:
            LH += self.gtr.sequence_logLH(self.tree.root.cseq, pattern_multiplicity=self.multiplicity)
        return LH


def convert_dates(self):
        '''
        This function converts the estimated "time_before_present" properties of all nodes
        to numerical dates stored in the "numdate" attribute. This date is further converted
        into a human readable date string in format %Y-%m-%d assuming the usual calendar.

        Returns
        -------
         None
            All manipulations are done in place on the tree

        '''
        from datetime import datetime, timedelta
        now = numeric_date()
        for node in self.tree.find_clades():
            years_bp = self.date2dist.to_years(node.time_before_present)
            if years_bp < 0 and self.real_dates:
                if not hasattr(node, "bad_branch") or node.bad_branch is False:
                    self.logger("ClockTree.convert_dates -- WARNING: The node is later than today, but it is not "
                        "marked as \"BAD\", which indicates the error in the "
                        "likelihood optimization.",4 , warn=True)
                else:
                    self.logger("ClockTree.convert_dates -- WARNING: node which is marked as \"BAD\" optimized "
                        "later than present day",4 , warn=True)

            node.numdate = now - years_bp

            # set the human-readable date
            year = np.floor(node.numdate)
            days = max(0,365.25 * (node.numdate - year)-1)
            try:  # datetime will only operate on dates after 1900
                n_date = datetime(year, 1, 1) + timedelta(days=days)
                node.date = datetime.strftime(n_date, "%Y-%m-%d")
            except:
                # this is the approximation not accounting for gap years etc
                n_date = datetime(1900, 1, 1) + timedelta(days=days)
                node.date = "%04d-%02d-%02d"%(year, n_date.month, n_date.day)

This function converts the estimated time_before_present properties of all nodes to numerical dates stored in the numdate attribute . This date is further converted into a human readable date string in format %Y - %m - %d assuming the usual calendar .


def date_uncertainty_due_to_rate(self, node, interval=(0.05, 0.095)):
        """use previously calculated variation of the rate to estimate
        the uncertainty in a particular numdate due to rate variation.

        Parameters
        ----------
        node : PhyloTree.Clade
            node for which the confidence interval is to be calculated
        interval : tuple, optional
            Array of length two, or tuple, defining the bounds of the confidence interval

        """
        if hasattr(node, "numdate_rate_variation"):
            from scipy.special import erfinv
            nsig = [np.sqrt(2.0)*erfinv(-1.0 + 2.0*x) if x*(1.0-x) else 0
                    for x in interval]
            l,c,u = [x[1] for x in node.numdate_rate_variation]
            return np.array([c + x*np.abs(y-c) for x,y in zip(nsig, (l,u))])

        else:
            return None

def date_uncertainty_due_to_rate ( self , node , interval = ( 0.05 , 0.095 ) ) : if hasattr ( node , "numdate_rate_variation" ) : from scipy . special import erfinv nsig = [ np . sqrt ( 2.0 ) * erfinv ( - 1.0 + 2.0 * x ) if x * ( 1.0 - x ) else 0 for x in interval ] l , c , u = [ x [ 1 ] for x in node . numdate_rate_variation ] return np . array ( [ c + x * np . abs ( y - c ) for x , y in zip ( nsig , ( l , u ) ) ] ) else : return None

use previously calculated variation of the rate to estimate the uncertainty in a particular numdate due to rate variation .


def get_max_posterior_region(self, node, fraction = 0.9):
        '''
        If temporal reconstruction was done using the marginal ML mode, the entire distribution of
        times is available. This function determines the interval around the highest
        posterior probability region that contains the specified fraction of the probability mass.
        In absense of marginal reconstruction, it will return uncertainty based on rate
        variation. If both are present, the wider interval will be returned.

        Parameters
        ----------

         node : PhyloTree.Clade
            The node for which the posterior region is to be calculated

         interval : float
            Float specifying who much of the posterior probability is
            to be contained in the region

        Returns
        -------
         max_posterior_region : numpy array
            Array with two numerical dates delineating the high posterior region

        '''
        if node.marginal_inverse_cdf=="delta":
            return np.array([node.numdate, node.numdate])

        min_max = (node.marginal_pos_LH.xmin, node.marginal_pos_LH.xmax)
        min_date, max_date = [self.date2dist.to_numdate(x) for x in min_max][::-1]
        if node.marginal_pos_LH.peak_pos == min_max[0]: #peak on the left
            return self.get_confidence_interval(node, (0, fraction))
        elif node.marginal_pos_LH.peak_pos == min_max[1]: #peak on the right
            return self.get_confidence_interval(node, (1.0-fraction ,1.0))
        else: # peak in the center of the distribution
            rate_contribution = self.date_uncertainty_due_to_rate(node, ((1-fraction)*0.5, 1.0-(1.0-fraction)*0.5))

            # construct height to position interpolators left and right of the peak
            # this assumes there is only one peak --- might fail in odd cases
            from scipy.interpolate import interp1d
            from scipy.optimize import minimize_scalar as minimize
            pidx = np.argmin(node.marginal_pos_LH.y)
            pval = np.min(node.marginal_pos_LH.y)
            left =  interp1d(node.marginal_pos_LH.y[:(pidx+1)]-pval, node.marginal_pos_LH.x[:(pidx+1)],
                            kind='linear', fill_value=min_max[0], bounds_error=False)
            right = interp1d(node.marginal_pos_LH.y[pidx:]-pval, node.marginal_pos_LH.x[pidx:],
                            kind='linear', fill_value=min_max[1], bounds_error=False)

            # function to minimize -- squared difference between prob mass and desired fracion
            def func(x, thres):
                interval = np.array([left(x), right(x)]).squeeze()
                return (thres - np.diff(node.marginal_cdf(np.array(interval))))**2

            # minimza and determine success
            sol = minimize(func, bracket=[0,10], args=(fraction,))
            if sol['success']:
                mutation_contribution = self.date2dist.to_numdate(np.array([right(sol['x']), left(sol['x'])]).squeeze())
            else: # on failure, return standard confidence interval
                mutation_contribution = None

            return self.combine_confidence(node.numdate, (min_date, max_date),
                                      c1=rate_contribution, c2=mutation_contribution)

If temporal reconstruction was done using the marginal ML mode the entire distribution of times is available . This function determines the interval around the highest posterior probability region that contains the specified fraction of the probability mass . In absense of marginal reconstruction it will return uncertainty based on rate variation . If both are present the wider interval will be returned .


def min_interp(interp_object):
    """
    Find the global minimum of a function represented as an interpolation object.
    """
    try:
        return interp_object.x[interp_object(interp_object.x).argmin()]
    except Exception as e:
        s = "Cannot find minimum of the interpolation object" + str(interp_object.x) + \
        "Minimal x: " + str(interp_object.x.min()) + "Maximal x: " + str(interp_object.x.max())
        raise e

Find the global minimum of a function represented as an interpolation object .


def median_interp(interp_object):
    """
    Find the median of the function represented as an interpolation object.
    """
    new_grid = np.sort(np.concatenate([interp_object.x[:-1] + 0.1*ii*np.diff(interp_object.x)
                                       for ii in range(10)]).flatten())

    tmp_prop = np.exp(-(interp_object(new_grid)-interp_object.y.min()))
    tmp_cumsum = np.cumsum(0.5*(tmp_prop[1:]+tmp_prop[:-1])*np.diff(new_grid))
    median_index = min(len(tmp_cumsum)-3, max(2,np.searchsorted(tmp_cumsum, tmp_cumsum[-1]*0.5)+1))
    return new_grid[median_index]

def median_interp ( interp_object ) : new_grid = np . sort ( np . concatenate ( [ interp_object . x [ : - 1 ] + 0.1 * ii * np . diff ( interp_object . x ) for ii in range ( 10 ) ] ) . flatten ( ) ) tmp_prop = np . exp ( - ( interp_object ( new_grid ) - interp_object . y . min ( ) ) ) tmp_cumsum = np . cumsum ( 0.5 * ( tmp_prop [ 1 : ] + tmp_prop [ : - 1 ] ) * np . diff ( new_grid ) ) median_index = min ( len ( tmp_cumsum ) - 3 , max ( 2 , np . searchsorted ( tmp_cumsum , tmp_cumsum [ - 1 ] * 0.5 ) + 1 ) ) return new_grid [ median_index ]

Find the median of the function represented as an interpolation object .


def numeric_date(dt=None):
    """
    Convert datetime object to the numeric date.
    The numeric date format is YYYY.F, where F is the fraction of the year passed

    Parameters
    ----------
     dt:  datetime.datetime, None
        date of to be converted. if None, assume today

    """
    if dt is None:
        dt = datetime.datetime.now()

    try:
        res = dt.year + dt.timetuple().tm_yday / 365.25
    except:
        res = None

    return res

def numeric_date ( dt = None ) : if dt is None : dt = datetime . datetime . now ( ) try : res = dt . year + dt . timetuple ( ) . tm_yday / 365.25 except : res = None return res

Convert datetime object to the numeric date . The numeric date format is YYYY . F where F is the fraction of the year passed


def from_regression(cls, clock_model):
        """
        Create the conversion object automatically from the tree

        Parameters
        ----------

         clock_model : dict
            dictionary as returned from TreeRegression with fields intercept and slope

        """
        dc = cls()
        dc.clock_rate = clock_model['slope']
        dc.intercept = clock_model['intercept']
        dc.chisq = clock_model['chisq'] if 'chisq' in clock_model else None
        dc.valid_confidence = clock_model['valid_confidence'] if 'valid_confidence' in clock_model else False
        if 'cov' in clock_model and dc.valid_confidence:
            dc.cov = clock_model['cov']
        dc.r_val = clock_model['r_val']
        return dc

def from_regression ( cls , clock_model ) : dc = cls ( ) dc . clock_rate = clock_model [ 'slope' ] dc . intercept = clock_model [ 'intercept' ] dc . chisq = clock_model [ 'chisq' ] if 'chisq' in clock_model else None dc . valid_confidence = clock_model [ 'valid_confidence' ] if 'valid_confidence' in clock_model else False if 'cov' in clock_model and dc . valid_confidence : dc . cov = clock_model [ 'cov' ] dc . r_val = clock_model [ 'r_val' ] return dc


def client(self):
        """
        Socket connection.
        """
        if not self._client:
            self._client = socket.create_connection(
                (self.host, self.port), self.timeout)
            self.logger.debug('Client connected with guacd server (%s, %s, %s)'
                              % (self.host, self.port, self.timeout))

        return self._client


def close(self):
        """
        Terminate connection with Guacamole guacd server.
        """
        self.client.close()
        self._client = None
        self.connected = False
        self.logger.debug('Connection closed.')

def
close
(
self
)
:
self
.
client
.
close
(
)
self
.
_client
=
self
.
connected
=
False
self
.
logger
.
debug
(
'Connection closed.'
)

Terminate connection with Guacamole guacd server .


def receive(self):
        """
        Receive instructions from Guacamole guacd server.
        """
        start = 0

        while True:
            idx = self._buffer.find(INST_TERM.encode(), start)
            if idx != -1:
                # instruction was fully received!
                line = self._buffer[:idx + 1].decode()
                self._buffer = self._buffer[idx + 1:]
                self.logger.debug('Received instruction: %s' % line)
                return line
            else:
                start = len(self._buffer)
                # we are still waiting for instruction termination
                buf = self.client.recv(BUF_LEN)
                if not buf:
                    # No data recieved, connection lost?!
                    self.close()
                    self.logger.debug(
                        'Failed to receive instruction. Closing.')
                    return None
                self._buffer.extend(buf)

Receive instructions from Guacamole guacd server .


def send(self, data):
        """
        Send encoded instructions to Guacamole guacd server.
        """
        self.logger.debug('Sending data: %s' % data)
        self.client.sendall(data.encode())

def
send
(
self
,
data
)
:
self
.
logger
.
debug
(
'Sending data: %s'
%
data
)
self
.
client
.
sendall
(
data
.
encode
(
)
)

Send encoded instructions to Guacamole guacd server .


def send_instruction(self, instruction):
        """
        Send instruction after encoding.
        """
        self.logger.debug('Sending instruction: %s' % str(instruction))
        return self.send(instruction.encode())

def
send_instruction
(
self
,
instruction
)
:
self
.
logger
.
debug
(
'Sending instruction: %s'
%
str
(
instruction
)
)
return
self
.
send
(
instruction
.
encode
(
)
)


def handshake(self, protocol='vnc', width=1024, height=768, dpi=96,
                  audio=None, video=None, image=None, **kwargs):
        """
        Establish connection with Guacamole guacd server via handshake.
        """
        if protocol not in PROTOCOLS:
            self.logger.debug('Invalid protocol: %s' % protocol)
            raise GuacamoleError('Cannot start Handshake. Missing protocol.')

        if audio is None:
            audio = list()

        if video is None:
            video = list()

        if image is None:
            image = list()

        # 1. Send 'select' instruction
        self.logger.debug('Send `select` instruction.')
        self.send_instruction(Instruction('select', protocol))

        # 2. Receive `args` instruction
        instruction = self.read_instruction()
        self.logger.debug('Expecting `args` instruction, received: %s'
                          % str(instruction))

        if not instruction:
            self.close()
            raise GuacamoleError(
                'Cannot establish Handshake. Connection Lost!')

        if instruction.opcode != 'args':
            self.close()
            raise GuacamoleError(
                'Cannot establish Handshake. Expected opcode `args`, '
                'received `%s` instead.' % instruction.opcode)

        # 3. Respond with size, audio & video support
        self.logger.debug('Send `size` instruction (%s, %s, %s)'
                          % (width, height, dpi))
        self.send_instruction(Instruction('size', width, height, dpi))

        self.logger.debug('Send `audio` instruction (%s)' % audio)
        self.send_instruction(Instruction('audio', *audio))

        self.logger.debug('Send `video` instruction (%s)' % video)
        self.send_instruction(Instruction('video', *video))

        self.logger.debug('Send `image` instruction (%s)' % image)
        self.send_instruction(Instruction('image', *image))

        # 4. Send `connect` instruction with proper values
        connection_args = [
            kwargs.get(arg.replace('-', '_'), '') for arg in instruction.args
        ]

        self.logger.debug('Send `connect` instruction (%s)' % connection_args)
        self.send_instruction(Instruction('connect', *connection_args))

        # 5. Receive ``ready`` instruction, with client ID.
        instruction = self.read_instruction()
        self.logger.debug('Expecting `ready` instruction, received: %s'
                          % str(instruction))

        if instruction.opcode != 'ready':
            self.logger.warning(
                'Expected `ready` instruction, received: %s instead')

        if instruction.args:
            self._id = instruction.args[0]
            self.logger.debug(
                'Established connection with client id: %s' % self.id)

        self.logger.debug('Handshake completed.')
        self.connected = True

Establish connection with Guacamole guacd server via handshake .


def utf8(unicode_str):
    """
    Return a utf-8 encoded string from a valid unicode string.

    :param unicode_str: Unicode string.

    :return: str
    """
    if six.PY2 and isinstance(unicode_str, __unicode__):
        return unicode_str.encode('utf-8')

    return unicode_str

def utf8 ( unicode_str ) : if six . PY2 and isinstance ( unicode_str , __unicode__ ) : return unicode_str . encode ( 'utf-8' ) return unicode_str

Return a utf - 8 encoded string from a valid unicode string .