tc/core/polyhedral/cuda/memory_promotion_heuristic.cc
// Assuming the mapping to threads happens in inverse order, i.e. the innermost
// loop is mapped to thread x, promote below that depth.
Assuming the mapping to threads happens in inverse order, i.e. the innermost
loop is mapped to thread x, promote below that depth.
Assuming the mapping to threads happens in inverse order i . e . the innermost loop is mapped to thread x promote below that depth .
void promoteToRegistersBelowThreads(
    Scop& scop,
    const ThreadIdxXScheduleDepthState& threadIdxXScheduleDepthState,
    size_t nRegisters) {
  using namespace tc::polyhedral::detail;
  auto root = scop.scheduleRoot();
  auto fullSched = fullSchedule(root);
  for (const auto& kvp : threadIdxXScheduleDepthState) {
    auto depth = kvp.second + 1;
    auto subdomain = kvp.first;
    auto bands = bandsContainingScheduleDepth(root, depth);
    if (bands.size() == 0) {
      continue;
    }
    std::function<bool(ScheduleTree*)> keepActive =
        [root, subdomain](const ScheduleTree* tree) {
          isl::union_set active = activeDomainPoints(root, tree);
          return !active.intersect(subdomain).is_empty();
        };
    bands = functional::Filter(keepActive, bands);
    bands = bandsSplitAfterDepth(bands, root, depth);
    for (auto band : bands) {
      auto points = activeDomainPoints(root, band);
      auto partialSched = partialSchedule(root, band);
      size_t nMappedThreads = 0;
      for (int j = 0; j < points.dim(isl::dim_type::param); ++j) {
        auto id = points.get_space().get_dim_id(isl::dim_type::param, j);
        for (size_t i = 0; i < mapping::ThreadId::kMaxDim; ++i) {
          if (id != mapping::ThreadId::makeId(i)) {
            continue;
          }
          if (isl::getParamValIfFixed(points, j) ==
              isl::val::zero(points.get_ctx())) {
            continue;
          }
          ++nMappedThreads;
          break;
        }
      }
      auto groupMap = TensorReferenceGroup::accessedBySubtree(band, scop);
      for (auto& tensorGroups : groupMap) {
        auto tensorId = tensorGroups.first;
        for (auto& group : tensorGroups.second) {
          auto sizes = group->approximationSizes();
          if (sizes.size() == 0) {
            continue;
          }
          if (!isPromotableToRegisterBelowThreads(
                  threadIdxXScheduleDepthState,
                  *group,
                  fullSched,
                  nMappedThreads,
                  points)) {
            continue;
          }
          if (!hasReuse(*group, fullSched, depth)) {
            continue;
          }
          scop.promoteGroup(
              Scop::PromotedDecl::Kind::Register,
              tensorId,
              std::move(group),
              band,
              partialSched);
        }
      }
    }
  }
}
void promoteToRegistersBelowThreads ( Scop & scop , const ThreadIdxXScheduleDepthState & threadIdxXScheduleDepthState , size_t nRegisters ) { using namespace tc :: polyhedral :: detail ; auto root = scop . scheduleRoot ( ) ; auto fullSched = fullSchedule ( root ) ; for ( const auto & kvp : threadIdxXScheduleDepthState ) { auto depth = kvp . second + 1 ; auto subdomain = kvp . first ; auto bands = bandsContainingScheduleDepth ( root , depth ) ; if ( bands . size ( ) == 0 ) { continue ; } std :: function < bool ( ScheduleTree * ) > keepActive = [ root , subdomain ] ( const ScheduleTree * tree ) { isl :: union_set active = activeDomainPoints ( root , tree ) ; return ! active . intersect ( subdomain ) . is_empty ( ) ; } ; bands = functional :: Filter ( keepActive , bands ) ; bands = bandsSplitAfterDepth ( bands , root , depth ) ; for ( auto band : bands ) { auto points = activeDomainPoints ( root , band ) ; auto partialSched = partialSchedule ( root , band ) ; size_t nMappedThreads = 0 ; for ( int j = 0 ; j < points . dim ( isl :: dim_type :: param ) ; ++ j ) { auto id = points . get_space ( ) . get_dim_id ( isl :: dim_type :: param , j ) ; for ( size_t i = 0 ; i < mapping :: ThreadId :: kMaxDim ; ++ i ) { if ( id != mapping :: ThreadId :: makeId ( i ) ) { continue ; } if ( isl :: getParamValIfFixed ( points , j ) == isl :: val :: zero ( points . get_ctx ( ) ) ) { continue ; } ++ nMappedThreads ; break ; } } auto groupMap = TensorReferenceGroup :: accessedBySubtree ( band , scop ) ; for ( auto & tensorGroups : groupMap ) { auto tensorId = tensorGroups . first ; for ( auto & group : tensorGroups . second ) { auto sizes = group -> approximationSizes ( ) ; if ( sizes . size ( ) == 0 ) { continue ; } if ( ! isPromotableToRegisterBelowThreads ( threadIdxXScheduleDepthState , * group , fullSched , nMappedThreads , points ) ) { continue ; } if ( ! hasReuse ( * group , fullSched , depth ) ) { continue ; } scop . promoteGroup ( Scop :: PromotedDecl :: Kind :: Register , tensorId , std :: move ( group ) , band , partialSched ) ; } } } } }
Assuming the mapping to threads happens in inverse order, i.e.
Assuming the mapping to threads happens in inverse order i . e .
// Collect all bands where a member is located at the given depth. // We may have no band members mapped to thread x in case when we // force-mapped everything to one thread. // Keep only those bands for which this depth was recorded. // Make sure the band ends at thread x depth so we can promote below it. // Find out how many threads are actually mapped.  Active domain points // will involve all mapping parameters when we take them below the // mapping.  Skip mapping parameters obviously mapped to 0, because they // do not correspond to band members that should be fixed to obtain // per-thread-group access relations. // TODO: sorting of groups and counting the number of promoted elements // No point in promoting a scalar that will go to a register anyway. // TODO: if something is already in shared, but reuse it within one // thread only, there is no point in keeping it in shared _if_ it // gets promoted into a register.