def trueQValues(Ts, R, discountFactor, policy):
 T = collapsedTransitions(Ts, policy)
 V = trueValues(T, R, discountFactor)
 Vnext = ((V * discountFactor) + R)
 numA = len(Ts)
 dim = len(R)
 Qs = zeros((dim, numA))
 for si in range(dim):
  for a in range(numA):
   Qs[(si, a)] = dot(Ts[a][si], Vnext)
 return Qs

def trueQValues ( Ts , R , discountFactor , policy ) : T = collapsedTransitions ( Ts , policy ) V = trueValues ( T , R , discountFactor ) Vnext = ( ( V * discountFactor ) + R ) numA = len ( Ts ) dim = len ( R ) Qs = zeros ( ( dim , numA ) ) for si in range ( dim ) : for a in range ( numA ) : Qs [ ( si , a ) ] = dot ( Ts [ a ] [ si ] , Vnext ) return Qs
the true q-values .