tc/core/polyhedral/cuda/memory_promotion_heuristic.cc
/*
 * For every place in the schedule tree where schedule depth (i.e., the number
 * of preceding band members) is "depth", promote tensor reference groups to
 * shared memory.  Split bands if necessary to insert promotions.
 *
 * Use at most "maxMemory" bytes.  If a groups does not fit the remaining
 * memory, do not promote it and keep looking for a smaller group.
 *
 * Only promote if the tensor elements referenced by the group are reused or
 * accessed in a non-coalesced way.
 */
For every place in the schedule tree where schedule depth  is "depth", promote tensor reference groups to
shared memory.  Split bands if necessary to insert promotions.

Use at most "maxMemory" bytes.  If a groups does not fit the remaining
memory, do not promote it and keep looking for a smaller group.

Only promote if the tensor elements referenced by the group are reused or
accessed in a non-coalesced way.
For every place in the schedule tree where schedule depth is " depth " promote tensor reference groups to shared memory . Split bands if necessary to insert promotions . Use at most " maxMemory " bytes . If a groups does not fit the remaining memory do not promote it and keep looking for a smaller group . Only promote if the tensor elements referenced by the group are reused or accessed in a non - coalesced way .
void promoteToSharedGreedy(
    Scop& scop,
    const ThreadIdxXScheduleDepthState& threadIdxXScheduleDepthState,
    const Block& block,
    size_t depth,
    size_t maxMemory) {
  using namespace tc::polyhedral::detail;
  if (depth == 0) {
    throw promotion::PromotionNYI("promotion before any band");
  }
  auto root = scop.scheduleRoot();
  auto bands = bandsContainingScheduleDepth(root, depth);
  bands = bandsSplitAfterDepth(bands, root, depth);
  auto fullSched = fullSchedule(root);
  size_t remainingMemory = maxMemory;
  for (auto bandNode : bands) {
    auto groupMap = TensorReferenceGroup::accessedBySubtree(bandNode, scop);
    auto partialSched = partialSchedule(root, bandNode);
    auto activePoints = activeDomainPoints(root, bandNode);
    using TensorGroupList = std::pair<isl::id, TensorGroupsInfo>;
    std::vector<TensorGroupList> groupLists(
        std::make_move_iterator(groupMap.begin()),
        std::make_move_iterator(groupMap.end()));
    auto refsCount = [](const TensorGroupsInfo& info) {
      size_t refs = 0;
      for (auto const& group : info) {
        refs += group->referenceIds().size();
      }
      return refs;
    };
    std::sort(
        groupLists.begin(),
        groupLists.end(),
        [refsCount](const TensorGroupList& l1, const TensorGroupList& l2) {
          auto r1 = refsCount(l1.second);
          auto r2 = refsCount(l2.second);
          return r1 == r2 ? l1.first.get_name() < l2.first.get_name() : r1 < r2;
        });
    for (auto& tensorGroups : groupLists) {
      auto tensorId = tensorGroups.first;
      std::sort(
          tensorGroups.second.begin(),
          tensorGroups.second.end(),
          [refsCount](
              const std::unique_ptr<TensorReferenceGroup>& group1,
              const std::unique_ptr<TensorReferenceGroup>& group2) {
            return group1->referenceIds().size() >
                group2->referenceIds().size();
          });
      for (auto& group : tensorGroups.second) {
        auto sizes = group->approximationSizes();
        if (sizes.size() == 0) {
          throw promotion::PromotionLogicError("cannot promote a scalar");
        }
        if (sizes.back() % 2 == 0) {
          sizes.back() += 1;
        }
        auto nApproximationElements = std::accumulate(
            sizes.begin(), sizes.end(), 1, std::multiplies<size_t>());
        auto memoryRequirement =
            nApproximationElements * scop.findArgument(tensorId).type().bytes();
        if (memoryRequirement > remainingMemory) {
          continue;
        }
        if (!hasReuse(*group, fullSched, depth) &&
            isCoalesced(
                threadIdxXScheduleDepthState,
                *group,
                fullSched,
                activePoints)) {
          continue;
        }
        scop.promoteGroup(
            Scop::PromotedDecl::Kind::SharedMem,
            tensorId,
            std::move(group),
            bandNode,
            partialSched,
            true);
        remainingMemory -= memoryRequirement;
      }
    }
    scop.insertSyncsAroundCopies(bandNode);
  }
}
void promoteToSharedGreedy ( Scop & scop , const ThreadIdxXScheduleDepthState & threadIdxXScheduleDepthState , const Block & block , size_t depth , size_t maxMemory ) { using namespace tc :: polyhedral :: detail ; if ( depth == 0 ) { throw promotion :: PromotionNYI ( " " ) ; } auto root = scop . scheduleRoot ( ) ; auto bands = bandsContainingScheduleDepth ( root , depth ) ; bands = bandsSplitAfterDepth ( bands , root , depth ) ; auto fullSched = fullSchedule ( root ) ; size_t remainingMemory = maxMemory ; for ( auto bandNode : bands ) { auto groupMap = TensorReferenceGroup :: accessedBySubtree ( bandNode , scop ) ; auto partialSched = partialSchedule ( root , bandNode ) ; auto activePoints = activeDomainPoints ( root , bandNode ) ; using TensorGroupList = std :: pair < isl :: id , TensorGroupsInfo > ; std :: vector < TensorGroupList > groupLists ( std :: make_move_iterator ( groupMap . begin ( ) ) , std :: make_move_iterator ( groupMap . end ( ) ) ) ; auto refsCount = [ ] ( const TensorGroupsInfo & info ) { size_t refs = 0 ; for ( auto const & group : info ) { refs += group -> referenceIds ( ) . size ( ) ; } return refs ; } ; std :: sort ( groupLists . begin ( ) , groupLists . end ( ) , [ refsCount ] ( const TensorGroupList & l1 , const TensorGroupList & l2 ) { auto r1 = refsCount ( l1 . second ) ; auto r2 = refsCount ( l2 . second ) ; return r1 == r2 ? l1 . first . get_name ( ) < l2 . first . get_name ( ) : r1 < r2 ; } ) ; for ( auto & tensorGroups : groupLists ) { auto tensorId = tensorGroups . first ; std :: sort ( tensorGroups . second . begin ( ) , tensorGroups . second . end ( ) , [ refsCount ] ( const std :: unique_ptr < TensorReferenceGroup > & group1 , const std :: unique_ptr < TensorReferenceGroup > & group2 ) { return group1 -> referenceIds ( ) . size ( ) > group2 -> referenceIds ( ) . size ( ) ; } ) ; for ( auto & group : tensorGroups . second ) { auto sizes = group -> approximationSizes ( ) ; if ( sizes . size ( ) == 0 ) { throw promotion :: PromotionLogicError ( " " ) ; } if ( sizes . back ( ) % 2 == 0 ) { sizes . back ( ) += 1 ; } auto nApproximationElements = std :: accumulate ( sizes . begin ( ) , sizes . end ( ) , 1 , std :: multiplies < size_t > ( ) ) ; auto memoryRequirement = nApproximationElements * scop . findArgument ( tensorId ) . type ( ) . bytes ( ) ; if ( memoryRequirement > remainingMemory ) { continue ; } if ( ! hasReuse ( * group , fullSched , depth ) && isCoalesced ( threadIdxXScheduleDepthState , * group , fullSched , activePoints ) ) { continue ; } scop . promoteGroup ( Scop :: PromotedDecl :: Kind :: SharedMem , tensorId , std :: move ( group ) , bandNode , partialSched , true ) ; remainingMemory -= memoryRequirement ; } } scop . insertSyncsAroundCopies ( bandNode ) ; } }
For every place in the schedule tree where schedule depth (i.e., the number
 of preceding band members) is "depth", promote tensor reference groups to
 shared memory.
For every place in the schedule tree where schedule depth ( i . e . the number of preceding band members ) is " depth " promote tensor reference groups to shared memory .
// 1. Collect all bands with a member located at the given depth in the // overall schedule.  Make sure this is the last member of the band by // splitting off the subsequent members into a different band. // 2. Compute full schedule without mapping filters.  The filters would make // it impossible to test for coalescing by incrementing a member of a band as // only the values divisible by grid or block size pass through the filter. // 3. For each band that ends at "depth", take decisions about promotion // immediately below it in the tree.  In particular, promote if the // approximated footprint fits into the remaining memory, and the reference // group either features reuse or is accessed in a non-coalesced way, or // both. // Prepare groups for sorting, to have specified order necessary for // reproducibility and tests. // Computes the total number of references in all groups. // Sort by the total number of references, then by name.  Because names are // guarenteed to be unique, the order is total. // Sort the reference groups to prioritize groups with more references as // they are more likely to benefit from promotion. // Do not promote if the group features no reuse and is accessed in a // coalesced way.