def inference(self, kern_r, kern_c, Xr, Xc, Zr, Zc, likelihood, Y, qU_mean ,qU_var_r, qU_var_c):
        """
        The SVI-VarDTC inference
        """

        N, D, Mr, Mc, Qr, Qc = Y.shape[0], Y.shape[1], Zr.shape[0], Zc.shape[0], Zr.shape[1], Zc.shape[1]

        uncertain_inputs_r = isinstance(Xr, VariationalPosterior)
        uncertain_inputs_c = isinstance(Xc, VariationalPosterior)
        uncertain_outputs = isinstance(Y, VariationalPosterior)

        beta = 1./likelihood.variance

        psi0_r, psi1_r, psi2_r = self.gatherPsiStat(kern_r, Xr, Zr, uncertain_inputs_r)
        psi0_c, psi1_c, psi2_c = self.gatherPsiStat(kern_c, Xc, Zc, uncertain_inputs_c)

        #======================================================================
        # Compute Common Components
        #======================================================================

        Kuu_r = kern_r.K(Zr).copy()
        diag.add(Kuu_r, self.const_jitter)
        Lr = jitchol(Kuu_r)

        Kuu_c = kern_c.K(Zc).copy()
        diag.add(Kuu_c, self.const_jitter)
        Lc = jitchol(Kuu_c)

        mu, Sr, Sc = qU_mean, qU_var_r, qU_var_c
        LSr = jitchol(Sr)
        LSc = jitchol(Sc)

        LcInvMLrInvT = dtrtrs(Lc,dtrtrs(Lr,mu.T)[0].T)[0]
        LcInvPsi2_cLcInvT = backsub_both_sides(Lc, psi2_c,'right')
        LrInvPsi2_rLrInvT = backsub_both_sides(Lr, psi2_r,'right')
        LcInvLSc = dtrtrs(Lc, LSc)[0]
        LrInvLSr = dtrtrs(Lr, LSr)[0]
        LcInvScLcInvT = tdot(LcInvLSc)
        LrInvSrLrInvT = tdot(LrInvLSr)
        LcInvPsi1_cT = dtrtrs(Lc, psi1_c.T)[0]
        LrInvPsi1_rT = dtrtrs(Lr, psi1_r.T)[0]

        tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT = (LrInvPsi2_rLrInvT*LrInvSrLrInvT).sum()
        tr_LcInvPsi2_cLcInvT_LcInvScLcInvT = (LcInvPsi2_cLcInvT*LcInvScLcInvT).sum()
        tr_LrInvSrLrInvT = np.square(LrInvLSr).sum()
        tr_LcInvScLcInvT = np.square(LcInvLSc).sum()
        tr_LrInvPsi2_rLrInvT = np.trace(LrInvPsi2_rLrInvT)
        tr_LcInvPsi2_cLcInvT = np.trace(LcInvPsi2_cLcInvT)

        #======================================================================
        # Compute log-likelihood
        #======================================================================

        logL_A = - np.square(Y).sum() \
               - (LcInvMLrInvT.T.dot(LcInvPsi2_cLcInvT).dot(LcInvMLrInvT)*LrInvPsi2_rLrInvT).sum() \
               -  tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT* tr_LcInvPsi2_cLcInvT_LcInvScLcInvT \
               + 2 * (Y * LcInvPsi1_cT.T.dot(LcInvMLrInvT).dot(LrInvPsi1_rT)).sum() - psi0_c * psi0_r \
               + tr_LrInvPsi2_rLrInvT * tr_LcInvPsi2_cLcInvT

        logL = -N*D/2.*(np.log(2.*np.pi)-np.log(beta)) + beta/2.* logL_A \
               -Mc * (np.log(np.diag(Lr)).sum()-np.log(np.diag(LSr)).sum())  -Mr * (np.log(np.diag(Lc)).sum()-np.log(np.diag(LSc)).sum()) \
               - np.square(LcInvMLrInvT).sum()/2. - tr_LrInvSrLrInvT * tr_LcInvScLcInvT/2. + Mr*Mc/2.

        #======================================================================
        # Compute dL_dKuu
        #======================================================================

        tmp =  beta* LcInvPsi2_cLcInvT.dot(LcInvMLrInvT).dot(LrInvPsi2_rLrInvT).dot(LcInvMLrInvT.T) \
             + beta* tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvPsi2_cLcInvT.dot(LcInvScLcInvT) \
             - beta* LcInvMLrInvT.dot(LrInvPsi1_rT).dot(Y.T).dot(LcInvPsi1_cT.T) \
             - beta/2. * tr_LrInvPsi2_rLrInvT* LcInvPsi2_cLcInvT - Mr/2.*np.eye(Mc) \
             + tdot(LcInvMLrInvT)/2. + tr_LrInvSrLrInvT/2. * LcInvScLcInvT

        dL_dKuu_c = backsub_both_sides(Lc, tmp, 'left')
        dL_dKuu_c += dL_dKuu_c.T
        dL_dKuu_c *= 0.5

        tmp =  beta* LcInvMLrInvT.T.dot(LcInvPsi2_cLcInvT).dot(LcInvMLrInvT).dot(LrInvPsi2_rLrInvT) \
             + beta* tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvPsi2_rLrInvT.dot(LrInvSrLrInvT) \
             - beta* LrInvPsi1_rT.dot(Y.T).dot(LcInvPsi1_cT.T).dot(LcInvMLrInvT) \
             - beta/2. * tr_LcInvPsi2_cLcInvT * LrInvPsi2_rLrInvT - Mc/2.*np.eye(Mr) \
             + tdot(LcInvMLrInvT.T)/2. + tr_LcInvScLcInvT/2. * LrInvSrLrInvT

        dL_dKuu_r = backsub_both_sides(Lr, tmp, 'left')
        dL_dKuu_r += dL_dKuu_r.T
        dL_dKuu_r *= 0.5

        #======================================================================
        # Compute dL_dthetaL
        #======================================================================

        dL_dthetaL = -D*N*beta/2. - logL_A*beta*beta/2.

        #======================================================================
        # Compute dL_dqU
        #======================================================================

        tmp = -beta * LcInvPsi2_cLcInvT.dot(LcInvMLrInvT).dot(LrInvPsi2_rLrInvT)\
              + beta* LcInvPsi1_cT.dot(Y).dot(LrInvPsi1_rT.T) - LcInvMLrInvT

        dL_dqU_mean = dtrtrs(Lc, dtrtrs(Lr, tmp.T, trans=1)[0].T, trans=1)[0]

        LScInv = dtrtri(LSc)
        tmp = -beta/2.*tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvPsi2_cLcInvT -tr_LrInvSrLrInvT/2.*np.eye(Mc)
        dL_dqU_var_c = backsub_both_sides(Lc, tmp, 'left') + tdot(LScInv.T) * Mr/2.

        LSrInv = dtrtri(LSr)
        tmp = -beta/2.*tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvPsi2_rLrInvT -tr_LcInvScLcInvT/2.*np.eye(Mr)
        dL_dqU_var_r = backsub_both_sides(Lr, tmp, 'left') + tdot(LSrInv.T) * Mc/2.

        #======================================================================
        # Compute the Posterior distribution of inducing points p(u|Y)
        #======================================================================

        post = PosteriorMultioutput(LcInvMLrInvT=LcInvMLrInvT, LcInvScLcInvT=LcInvScLcInvT,
                LrInvSrLrInvT=LrInvSrLrInvT, Lr=Lr, Lc=Lc, kern_r=kern_r, Xr=Xr, Zr=Zr)

        #======================================================================
        # Compute dL_dpsi
        #======================================================================

        dL_dpsi0_r = - psi0_c * beta/2. * np.ones((D,))
        dL_dpsi0_c = - psi0_r * beta/2. * np.ones((N,))

        dL_dpsi1_c = beta * dtrtrs(Lc, (Y.dot(LrInvPsi1_rT.T).dot(LcInvMLrInvT.T)).T, trans=1)[0].T
        dL_dpsi1_r = beta * dtrtrs(Lr, (Y.T.dot(LcInvPsi1_cT.T).dot(LcInvMLrInvT)).T, trans=1)[0].T

        tmp = beta/2.*(-LcInvMLrInvT.dot(LrInvPsi2_rLrInvT).dot(LcInvMLrInvT.T) - tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvScLcInvT
              +tr_LrInvPsi2_rLrInvT *np.eye(Mc))
        dL_dpsi2_c = backsub_both_sides(Lc, tmp, 'left')
        tmp = beta/2.*(-LcInvMLrInvT.T.dot(LcInvPsi2_cLcInvT).dot(LcInvMLrInvT) - tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvSrLrInvT
              +tr_LcInvPsi2_cLcInvT *np.eye(Mr))
        dL_dpsi2_r = backsub_both_sides(Lr, tmp, 'left')

        if not uncertain_inputs_r:
            dL_dpsi1_r += psi1_r.dot(dL_dpsi2_r+dL_dpsi2_r.T)
        if not uncertain_inputs_c:
            dL_dpsi1_c += psi1_c.dot(dL_dpsi2_c+dL_dpsi2_c.T)

        grad_dict = {
            'dL_dthetaL':dL_dthetaL,
            'dL_dqU_mean':dL_dqU_mean,
            'dL_dqU_var_c':dL_dqU_var_c,
            'dL_dqU_var_r':dL_dqU_var_r,
            'dL_dKuu_c': dL_dKuu_c,
            'dL_dKuu_r': dL_dKuu_r,
        }

        if uncertain_inputs_c:
            grad_dict['dL_dpsi0_c'] = dL_dpsi0_c
            grad_dict['dL_dpsi1_c'] = dL_dpsi1_c
            grad_dict['dL_dpsi2_c'] = dL_dpsi2_c
        else:
            grad_dict['dL_dKdiag_c'] = dL_dpsi0_c
            grad_dict['dL_dKfu_c'] = dL_dpsi1_c

        if uncertain_inputs_r:
            grad_dict['dL_dpsi0_r'] = dL_dpsi0_r
            grad_dict['dL_dpsi1_r'] = dL_dpsi1_r
            grad_dict['dL_dpsi2_r'] = dL_dpsi2_r
        else:
            grad_dict['dL_dKdiag_r'] = dL_dpsi0_r
            grad_dict['dL_dKfu_r'] = dL_dpsi1_r

        return post, logL, grad_dict
def inference ( self , kern_r , kern_c , Xr , Xc , Zr , Zc , likelihood , Y , qU_mean , qU_var_r , qU_var_c ) : N , D , Mr , Mc , Qr , Qc = Y . shape [ 0 ] , Y . shape [ 1 ] , Zr . shape [ 0 ] , Zc . shape [ 0 ] , Zr . shape [ 1 ] , Zc . shape [ 1 ] uncertain_inputs_r = isinstance ( Xr , VariationalPosterior ) uncertain_inputs_c = isinstance ( Xc , VariationalPosterior ) uncertain_outputs = isinstance ( Y , VariationalPosterior ) beta = 1. / likelihood . variance psi0_r , psi1_r , psi2_r = self . gatherPsiStat ( kern_r , Xr , Zr , uncertain_inputs_r ) psi0_c , psi1_c , psi2_c = self . gatherPsiStat ( kern_c , Xc , Zc , uncertain_inputs_c ) Kuu_r = kern_r . K ( Zr ) . copy ( ) diag . add ( Kuu_r , self . const_jitter ) Lr = jitchol ( Kuu_r ) Kuu_c = kern_c . K ( Zc ) . copy ( ) diag . add ( Kuu_c , self . const_jitter ) Lc = jitchol ( Kuu_c ) mu , Sr , Sc = qU_mean , qU_var_r , qU_var_c LSr = jitchol ( Sr ) LSc = jitchol ( Sc ) LcInvMLrInvT = dtrtrs ( Lc , dtrtrs ( Lr , mu . T ) [ 0 ] . T ) [ 0 ] LcInvPsi2_cLcInvT = backsub_both_sides ( Lc , psi2_c , 'right' ) LrInvPsi2_rLrInvT = backsub_both_sides ( Lr , psi2_r , 'right' ) LcInvLSc = dtrtrs ( Lc , LSc ) [ 0 ] LrInvLSr = dtrtrs ( Lr , LSr ) [ 0 ] LcInvScLcInvT = tdot ( LcInvLSc ) LrInvSrLrInvT = tdot ( LrInvLSr ) LcInvPsi1_cT = dtrtrs ( Lc , psi1_c . T ) [ 0 ] LrInvPsi1_rT = dtrtrs ( Lr , psi1_r . T ) [ 0 ] tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT = ( LrInvPsi2_rLrInvT * LrInvSrLrInvT ) . sum ( ) tr_LcInvPsi2_cLcInvT_LcInvScLcInvT = ( LcInvPsi2_cLcInvT * LcInvScLcInvT ) . sum ( ) tr_LrInvSrLrInvT = np . square ( LrInvLSr ) . sum ( ) tr_LcInvScLcInvT = np . square ( LcInvLSc ) . sum ( ) tr_LrInvPsi2_rLrInvT = np . trace ( LrInvPsi2_rLrInvT ) tr_LcInvPsi2_cLcInvT = np . trace ( LcInvPsi2_cLcInvT ) logL_A = - np . square ( Y ) . sum ( ) - ( LcInvMLrInvT . T . dot ( LcInvPsi2_cLcInvT ) . dot ( LcInvMLrInvT ) * LrInvPsi2_rLrInvT ) . sum ( ) - tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * tr_LcInvPsi2_cLcInvT_LcInvScLcInvT + 2 * ( Y * LcInvPsi1_cT . T . dot ( LcInvMLrInvT ) . dot ( LrInvPsi1_rT ) ) . sum ( ) - psi0_c * psi0_r + tr_LrInvPsi2_rLrInvT * tr_LcInvPsi2_cLcInvT logL = - N * D / 2. * ( np . log ( 2. * np . pi ) - np . log ( beta ) ) + beta / 2. * logL_A - Mc * ( np . log ( np . diag ( Lr ) ) . sum ( ) - np . log ( np . diag ( LSr ) ) . sum ( ) ) - Mr * ( np . log ( np . diag ( Lc ) ) . sum ( ) - np . log ( np . diag ( LSc ) ) . sum ( ) ) - np . square ( LcInvMLrInvT ) . sum ( ) / 2. - tr_LrInvSrLrInvT * tr_LcInvScLcInvT / 2. + Mr * Mc / 2. tmp = beta * LcInvPsi2_cLcInvT . dot ( LcInvMLrInvT ) . dot ( LrInvPsi2_rLrInvT ) . dot ( LcInvMLrInvT . T ) + beta * tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvPsi2_cLcInvT . dot ( LcInvScLcInvT ) - beta * LcInvMLrInvT . dot ( LrInvPsi1_rT ) . dot ( Y . T ) . dot ( LcInvPsi1_cT . T ) - beta / 2. * tr_LrInvPsi2_rLrInvT * LcInvPsi2_cLcInvT - Mr / 2. * np . eye ( Mc ) + tdot ( LcInvMLrInvT ) / 2. + tr_LrInvSrLrInvT / 2. * LcInvScLcInvT dL_dKuu_c = backsub_both_sides ( Lc , tmp , 'left' ) dL_dKuu_c += dL_dKuu_c . T dL_dKuu_c *= 0.5 tmp = beta * LcInvMLrInvT . T . dot ( LcInvPsi2_cLcInvT ) . dot ( LcInvMLrInvT ) . dot ( LrInvPsi2_rLrInvT ) + beta * tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvPsi2_rLrInvT . dot ( LrInvSrLrInvT ) - beta * LrInvPsi1_rT . dot ( Y . T ) . dot ( LcInvPsi1_cT . T ) . dot ( LcInvMLrInvT ) - beta / 2. * tr_LcInvPsi2_cLcInvT * LrInvPsi2_rLrInvT - Mc / 2. * np . eye ( Mr ) + tdot ( LcInvMLrInvT . T ) / 2. + tr_LcInvScLcInvT / 2. * LrInvSrLrInvT dL_dKuu_r = backsub_both_sides ( Lr , tmp , 'left' ) dL_dKuu_r += dL_dKuu_r . T dL_dKuu_r *= 0.5 dL_dthetaL = - D * N * beta / 2. - logL_A * beta * beta / 2. tmp = - beta * LcInvPsi2_cLcInvT . dot ( LcInvMLrInvT ) . dot ( LrInvPsi2_rLrInvT ) + beta * LcInvPsi1_cT . dot ( Y ) . dot ( LrInvPsi1_rT . T ) - LcInvMLrInvT dL_dqU_mean = dtrtrs ( Lc , dtrtrs ( Lr , tmp . T , trans = 1 ) [ 0 ] . T , trans = 1 ) [ 0 ] LScInv = dtrtri ( LSc ) tmp = - beta / 2. * tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvPsi2_cLcInvT - tr_LrInvSrLrInvT / 2. * np . eye ( Mc ) dL_dqU_var_c = backsub_both_sides ( Lc , tmp , 'left' ) + tdot ( LScInv . T ) * Mr / 2. LSrInv = dtrtri ( LSr ) tmp = - beta / 2. * tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvPsi2_rLrInvT - tr_LcInvScLcInvT / 2. * np . eye ( Mr ) dL_dqU_var_r = backsub_both_sides ( Lr , tmp , 'left' ) + tdot ( LSrInv . T ) * Mc / 2. post = PosteriorMultioutput ( LcInvMLrInvT = LcInvMLrInvT , LcInvScLcInvT = LcInvScLcInvT , LrInvSrLrInvT = LrInvSrLrInvT , Lr = Lr , Lc = Lc , kern_r = kern_r , Xr = Xr , Zr = Zr ) dL_dpsi0_r = - psi0_c * beta / 2. * np . ones ( ( D , ) ) dL_dpsi0_c = - psi0_r * beta / 2. * np . ones ( ( N , ) ) dL_dpsi1_c = beta * dtrtrs ( Lc , ( Y . dot ( LrInvPsi1_rT . T ) . dot ( LcInvMLrInvT . T ) ) . T , trans = 1 ) [ 0 ] . T dL_dpsi1_r = beta * dtrtrs ( Lr , ( Y . T . dot ( LcInvPsi1_cT . T ) . dot ( LcInvMLrInvT ) ) . T , trans = 1 ) [ 0 ] . T tmp = beta / 2. * ( - LcInvMLrInvT . dot ( LrInvPsi2_rLrInvT ) . dot ( LcInvMLrInvT . T ) - tr_LrInvPsi2_rLrInvT_LrInvSrLrInvT * LcInvScLcInvT + tr_LrInvPsi2_rLrInvT * np . eye ( Mc ) ) dL_dpsi2_c = backsub_both_sides ( Lc , tmp , 'left' ) tmp = beta / 2. * ( - LcInvMLrInvT . T . dot ( LcInvPsi2_cLcInvT ) . dot ( LcInvMLrInvT ) - tr_LcInvPsi2_cLcInvT_LcInvScLcInvT * LrInvSrLrInvT + tr_LcInvPsi2_cLcInvT * np . eye ( Mr ) ) dL_dpsi2_r = backsub_both_sides ( Lr , tmp , 'left' ) if not uncertain_inputs_r : dL_dpsi1_r += psi1_r . dot ( dL_dpsi2_r + dL_dpsi2_r . T ) if not uncertain_inputs_c : dL_dpsi1_c += psi1_c . dot ( dL_dpsi2_c + dL_dpsi2_c . T ) grad_dict = { 'dL_dthetaL' : dL_dthetaL , 'dL_dqU_mean' : dL_dqU_mean , 'dL_dqU_var_c' : dL_dqU_var_c , 'dL_dqU_var_r' : dL_dqU_var_r , 'dL_dKuu_c' : dL_dKuu_c , 'dL_dKuu_r' : dL_dKuu_r , } if uncertain_inputs_c : grad_dict [ 'dL_dpsi0_c' ] = dL_dpsi0_c grad_dict [ 'dL_dpsi1_c' ] = dL_dpsi1_c grad_dict [ 'dL_dpsi2_c' ] = dL_dpsi2_c else : grad_dict [ 'dL_dKdiag_c' ] = dL_dpsi0_c grad_dict [ 'dL_dKfu_c' ] = dL_dpsi1_c if uncertain_inputs_r : grad_dict [ 'dL_dpsi0_r' ] = dL_dpsi0_r grad_dict [ 'dL_dpsi1_r' ] = dL_dpsi1_r grad_dict [ 'dL_dpsi2_r' ] = dL_dpsi2_r else : grad_dict [ 'dL_dKdiag_r' ] = dL_dpsi0_r grad_dict [ 'dL_dKfu_r' ] = dL_dpsi1_r return post , logL , grad_dict
The SVI-VarDTC inference