test/processingMeasures.cpp
/*

Current data dump of measures has the following content
------begin of the file
=================================
Rank 0 Region daxpy MEASURES 
features: [ 9027,  ]: policy: 1 , count: 1 , total: 0.000241501 , time_avg: 0.000241501
features: [ 38863,  ]: policy: 0 , count: 1 , total: 0.00714597 , time_avg: 0.00714597
features: [ 61640,  ]: policy: 0 , count: 1 , total: 0.00875109 , time_avg: 0.00875109
features: [ 148309,  ]: policy: 0 , count: 1 , total: 0.00658767 , time_avg: 0.00658767
features: [ 240700,  ]: policy: 0 , count: 1 , total: 0.00978215 , time_avg: 0.00978215
features: [ 366001,  ]: policy: 1 , count: 1 , total: 0.00142485 , time_avg: 0.00142485
..
.-  
Rank 0 Region daxpy Reduce 
features: [ 9027, ]: P:1 T: 0.000241501
features: [ 38863, ]: P:0 T: 0.00714597
 
---------end of file
We only need to read in the first portion to populate the map of feature vector+policy -> measure pair<count, total>

std::map< std::pair< std::vector<float>, int >, std::unique_ptr<Region::Measure> > measures;

 * */
Current data dump of measures has the following content
begin of the file



end of file
We only need to read in the first portion to populate the map of feature vector+policy -> measure pair


Current data dump of measures has the following content begin of the file end of file We only need to read in the first portion to populate the map of feature vector + policy - > measure pair
bool
Region::loadPreviousMeasures(const string & filename)
{
  string prev_data_file = filename;
  ifstream datafile (prev_data_file.c_str(), ios::in);
  if (!datafile.is_open())
  {
    cout<<"Region::loadPreviousMeasures() cannot find/open txt file "<< prev_data_file<<endl;
    return false;
  } 
  cout<<"loadPreviousMeasures() opens previous measure data file successfully: "<< prev_data_file <<endl;
  while (!datafile.eof())
  {
    string line;
    getline(datafile, line);
    if (line==".-")
    {
      break;
    }
    size_t pos=0; 
    while (pos<line.size()) 
    {
      pos=line.find('[', pos);
      if (pos==string::npos) 
        break; 
      pos++; 
      size_t end_bracket_pos=line.find(']', pos);
      if (end_bracket_pos==string::npos) 
      {
        cerr<<"Error in Region::loadPreviousMeasures(). cannot find ending ] in "<< line <<endl;
        return false; 
      }
      int cur_policy;
      size_t end=pos;
      vector<float> cur_features; 
      while (pos<end_bracket_pos)
      {
        string cell=extractOneCell(line, pos); 
        if (pos < end_bracket_pos)
        {
          cur_features.push_back(stof(cell));
        }
        else
          break; 
      }
     max_feature_count = max ( max_feature_count, cur_features.size());
     pos = line.find("policy:", end_bracket_pos);
      if (pos==string::npos)
        return false;
      pos+=7;
      string policy_id=extractOneCell(line, pos);
      cur_policy = stoi(policy_id);
      max_policy_id = max (max_policy_id, (size_t)cur_policy);
      int cur_count; 
      pos = line.find("count:", end_bracket_pos);
      if (pos==string::npos)
        return false;
      pos+=6;
      string count=extractOneCell(line, pos);
      cur_count = stoi(count);
      pos = line.find("total:", end_bracket_pos);
      if (pos==string::npos)
        return false;
      pos+=6;
      string total=extractOneCell(line, pos);
      float cur_total_time = stof(total); 
      auto iter = measures.find({cur_features, cur_policy});
      if (iter == measures.end()) {
        iter = measures
          .insert(std::make_pair(
                std::make_pair(cur_features, cur_policy),
                std::move(
                  std::make_unique<Region::Measure>(cur_count, cur_total_time))))
          .first;
      } else {
        assert (false);
        iter->second->exec_count+=cur_count;
        iter->second->time_total += cur_total_time;
      }
    } 
  } 
  cout<<"After insertion: measures.size():"<<measures.size() 
    <<" max_feature_count="<< max_feature_count
    <<" max_policy_id="<< max_policy_id <<endl;
  datafile.close();
  return true;
}
bool Region :: loadPreviousMeasures ( const string & filename ) { string prev_data_file = filename ; ifstream datafile ( prev_data_file . c_str ( ) , ios :: in ) ; if ( ! datafile . is_open ( ) ) { cout << " " << prev_data_file << endl ; return false ; } cout << " " << prev_data_file << endl ; while ( ! datafile . eof ( ) ) { string line ; getline ( datafile , line ) ; if ( line == " " ) { break ; } size_t pos = 0 ; while ( pos < line . size ( ) ) { pos = line . find ( ' ' , pos ) ; if ( pos == string :: npos ) break ; pos ++ ; size_t end_bracket_pos = line . find ( ' ' , pos ) ; if ( end_bracket_pos == string :: npos ) { cerr << " " << line << endl ; return false ; } int cur_policy ; size_t end = pos ; vector < float > cur_features ; while ( pos < end_bracket_pos ) { string cell = extractOneCell ( line , pos ) ; if ( pos < end_bracket_pos ) { cur_features . push_back ( stof ( cell ) ) ; } else break ; } max_feature_count = max ( max_feature_count , cur_features . size ( ) ) ; pos = line . find ( " " , end_bracket_pos ) ; if ( pos == string :: npos ) return false ; pos += 7 ; string policy_id = extractOneCell ( line , pos ) ; cur_policy = stoi ( policy_id ) ; max_policy_id = max ( max_policy_id , ( size_t ) cur_policy ) ; int cur_count ; pos = line . find ( " " , end_bracket_pos ) ; if ( pos == string :: npos ) return false ; pos += 6 ; string count = extractOneCell ( line , pos ) ; cur_count = stoi ( count ) ; pos = line . find ( " " , end_bracket_pos ) ; if ( pos == string :: npos ) return false ; pos += 6 ; string total = extractOneCell ( line , pos ) ; float cur_total_time = stof ( total ) ; auto iter = measures . find ( { cur_features , cur_policy } ) ; if ( iter == measures . end ( ) ) { iter = measures . insert ( std :: make_pair ( std :: make_pair ( cur_features , cur_policy ) , std :: move ( std :: make_unique < Region :: Measure > ( cur_count , cur_total_time ) ) ) ) . first ; } else { assert ( false ) ; iter -> second -> exec_count += cur_count ; iter -> second -> time_total += cur_total_time ; } } } cout << " " << measures . size ( ) << " " << max_feature_count << " " << max_policy_id << endl ; datafile . close ( ) ; return true ; }
Current data dump of measures has the following content
begin of the file
Current data dump of measures has the following content begin of the file
// cout<<"Found end line, stopping.."<<endl; // we have the line in a string anyway, just use it. // a line without [  // skip [ // something is wrong // extract policy // extract one or more features, separated by ',' between pos and end_bracket_pos // may reaching over ] , skip them // only keep one extracted before ']' //  cout<<stoi(cell) << " " ; // convert to integer, they are mostly sizes of things right now //cout<<"policy:"<<stoi(policy_id) <<" "; // convert to integer, they are mostly sizes of things right now // extract count //cout<<"count:"<<stoi(count) <<" "; // convert to integer, they are mostly sizes of things right now //extract total execution time // cout<<"total exe time:"<<stof(total) <<endl; // convert to integer, they are mostly sizes of things right now // we dont' really expect anything going this branch // the previous training data should already be aggregated data.  // hande a line  // end while