torch/csrc/jit/autodiff.cpp
// Any temporary value from the primal graphs needs to be captured for later use in the
// reverse graph, to avoid costly recomputations. However, a lot of the nodes we have
// in our graphs are simply constants, which are cheap to execute and replicate, and so
// it's better to just copy them into the reverse graph, without polluting the output
// lists unnecessarily.
Any temporary value from the primal graphs needs to be captured for later use in the
reverse graph, to avoid costly recomputations. However, a lot of the nodes we have
in our graphs are simply constants, which are cheap to execute and replicate, and so
it's better to just copy them into the reverse graph, without polluting the output
lists unnecessarily.
Any temporary value from the primal graphs needs to be captured for later use in the reverse graph to avoid costly recomputations . However a lot of the nodes we have in our graphs are simply constants which are cheap to execute and replicate and so it ' s better to just copy them into the reverse graph without polluting the output lists unnecessarily .
static void liftConstants(Gradient& grad_desc, ReverseDetails& rev_info) {
  static const auto err = [](Value*) -> Value* {
    throw std::runtime_error("unexpected input");
  };
  auto & graph = *grad_desc.f;
  Block* reverse_block = rev_info.reverse_block;
  for (Node *top_node : reverse_block->nodes()) {
    JIT_ASSERT(top_node->kind() == prim::GradOf ||
               top_node->kind() == prim::AutogradAdd ||
               top_node->kind() == prim::Undefined);
    if (top_node->kind() != prim::GradOf) continue;
    Block * grad_body = top_node->blocks().at(0);
    for (Node *node : grad_body->nodes()) {
      for (Value * input : node->inputs()) {
        if (input->node()->kind() != prim::Constant) continue;
        if (input->node()->owningBlock() == grad_body) continue;
        Node *lifted_constant = graph.createClone(input->node(), err);
        reverse_block->prependNode(lifted_constant);
        node->replaceInputWith(input, lifted_constant->output());
      }
    }
  }
  EliminateCommonSubexpression(reverse_block);
}
static void liftConstants ( Gradient & grad_desc , ReverseDetails & rev_info ) { static const auto err = [ ] ( Value * ) -> Value * { throw std :: runtime_error ( " " ) ; } ; auto & graph = * grad_desc . f ; Block * reverse_block = rev_info . reverse_block ; for ( Node * top_node : reverse_block -> nodes ( ) ) { JIT_ASSERT ( top_node -> kind ( ) == prim :: GradOf || top_node -> kind ( ) == prim :: AutogradAdd || top_node -> kind ( ) == prim :: Undefined ) ; if ( top_node -> kind ( ) != prim :: GradOf ) continue ; Block * grad_body = top_node -> blocks ( ) . at ( 0 ) ; for ( Node * node : grad_body -> nodes ( ) ) { for ( Value * input : node -> inputs ( ) ) { if ( input -> node ( ) -> kind ( ) != prim :: Constant ) continue ; if ( input -> node ( ) -> owningBlock ( ) == grad_body ) continue ; Node * lifted_constant = graph . createClone ( input -> node ( ) , err ) ; reverse_block -> prependNode ( lifted_constant ) ; node -> replaceInputWith ( input , lifted_constant -> output ( ) ) ; } } } EliminateCommonSubexpression ( reverse_block ) ; }
Any temporary value from the primal graphs needs to be captured for later use in the
reverse graph, to avoid costly recomputations.
Any temporary value from the primal graphs needs to be captured for later use in the reverse graph to avoid costly recomputations .
// It's possible the we've cloned the same constants many times, // so we use CSE to deduplicate them.