c++/EEDB/WebServices/UploadServer.cpp
//////////////////////////////////////////////////////////////////////////////////
//
//  new upload (non-form) based methods
//
//////////////////////////////////////////////////////////////////////////////////
new upload (non-form) based methods
new upload ( non - form ) based methods
void  EEDB::WebServices::UploadServer::prevent_duplicate_uploads() {
  if(!_userDB) { return; }
  if(!_user_profile) { return; }
  if(_parameters["check_duplicates"] != "true") { return; }
  if(_previous_upload_origfiles.empty()) {
    EEDB::SPStreams::FederatedSourceStream *stream = new EEDB::SPStreams::FederatedSourceStream();
    stream->set_peer_search_depth(10);
    stream->add_seed_peer(_user_profile->user_registry());
    stream->stream_data_sources();
    while(EEDB::DataSource *source = (EEDB::DataSource*)stream->next_in_stream()) {
      vector<Metadata*> mdlist = source->metadataset()->find_all_metadata_like("orig_filename", "");
      for(unsigned j=0; j<mdlist.size(); j++) {
        _previous_upload_origfiles[mdlist[j]->data()]++;
      }
    }
    fprintf(stderr, "loaded %ld previous_upload_origfiles\n", _previous_upload_origfiles.size());
  }
  map<string, long>::iterator it3;
  for(it3=_previous_upload_origfiles.begin(); it3!=_previous_upload_origfiles.end(); it3++) {
    if((*it3).second > 1) {
      fprintf(stderr, "%s duplicate origfile uploads [%s] %ld\n",
            _user_profile->email_identity().c_str(), (*it3).first.c_str(), (*it3).second);
    }
  }
  map<string, UploadFile>::iterator it1, it2;
  it1=_upload_file_list.begin();
  while(it1!=_upload_file_list.end()) {
    string orig_filename = it1->first;
    it2 = it1;
    it1++;
    if(_previous_upload_origfiles.find(orig_filename) != _previous_upload_origfiles.end()) {
      _upload_file_list.erase(it2);
      continue;
    }
    const char* sql = "SELECT job_id FROM job join job_2_metadata using(job_id) join metadata using(metadata_id) \
                       WHERE user_id=? and status in ('READY', 'RUN') and data_type ='original_file' \
                             and created > DATE_SUB(NOW(), INTERVAL 4 HOUR) and data=?";
    dynadata value = _userDB->fetch_col_value(sql, "ds",  _user_profile->primary_id(), orig_filename.c_str());
    if(value.type == MQDB::INT) {
      _upload_file_list.erase(it2);
    }
  }
  fprintf(stderr, "new uploads after duplicate check %ld\n",_upload_file_list.size());
}
void EEDB :: WebServices :: UploadServer :: prevent_duplicate_uploads ( ) { if ( ! _userDB ) { return ; } if ( ! _user_profile ) { return ; } if ( _parameters [ " " ] != " " ) { return ; } if ( _previous_upload_origfiles . empty ( ) ) { EEDB :: SPStreams :: FederatedSourceStream * stream = new EEDB :: SPStreams :: FederatedSourceStream ( ) ; stream -> set_peer_search_depth ( 10 ) ; stream -> add_seed_peer ( _user_profile -> user_registry ( ) ) ; stream -> stream_data_sources ( ) ; while ( EEDB :: DataSource * source = ( EEDB :: DataSource * ) stream -> next_in_stream ( ) ) { vector < Metadata * > mdlist = source -> metadataset ( ) -> find_all_metadata_like ( " " , " " ) ; for ( unsigned j = 0 ; j < mdlist . size ( ) ; j ++ ) { _previous_upload_origfiles [ mdlist [ j ] -> data ( ) ] ++ ; } } fprintf ( stderr , " \n " , _previous_upload_origfiles . size ( ) ) ; } map < string , long > :: iterator it3 ; for ( it3 = _previous_upload_origfiles . begin ( ) ; it3 != _previous_upload_origfiles . end ( ) ; it3 ++ ) { if ( ( * it3 ) . second > 1 ) { fprintf ( stderr , " \n " , _user_profile -> email_identity ( ) . c_str ( ) , ( * it3 ) . first . c_str ( ) , ( * it3 ) . second ) ; } } map < string , UploadFile > :: iterator it1 , it2 ; it1 = _upload_file_list . begin ( ) ; while ( it1 != _upload_file_list . end ( ) ) { string orig_filename = it1 -> first ; it2 = it1 ; it1 ++ ; if ( _previous_upload_origfiles . find ( orig_filename ) != _previous_upload_origfiles . end ( ) ) { _upload_file_list . erase ( it2 ) ; continue ; } const char * sql = " \
 \
 " ; dynadata value = _userDB -> fetch_col_value ( sql , " " , _user_profile -> primary_id ( ) , orig_filename . c_str ( ) ) ; if ( value . type == MQDB :: INT ) { _upload_file_list . erase ( it2 ) ; } } fprintf ( stderr , " \n " , _upload_file_list . size ( ) ) ; }
new upload (non-form) based methods
new upload ( non - form ) based methods
//check the _upload_file_list original_filename against previous upload jobs from this user with the same original_file //need to load the list of previous uploaded orig_filenames //EEDB::SPStreams::FederatedSourceStream *stream = EEDB::WebServices::WebBase::source_stream(); //if(source->classname() != EEDB::FeatureSource::class_name) { continue; } //first check for files which are really loaded in the system as DataSources //found duplicate so remove it2 //next check if it might be in the process of running //fprintf(stderr, "local_filename=[%s]\n", local_filename.c_str()); //found duplicate so remove it2