title: Node app crashing again and again due to mongo query
body: <p>this is my db collection document. </p>
  
  <pre><code>{
  "_id" : ObjectId("55e86e98f493590878bb45d7"),
  "RecordID" : 84096380,
  "FN" : "Michael",
  "MI" : "",
  "LN" : "Horn",
  "NAME_PRE" : "MR",
  "ADDR" : "160 Yankee Camp Rd",
  "CITY" : "Telford",
  "ST" : "TN",
  "ZIP" : 37690,
  "APT" : "",
  "Z4" : 2200,
  "DPC" : 605,
  "CAR_RTE" : "R001",
  "WALK_SEQ" : 228,
  "LOT" : "0136A",
  "FIPS_ST" : 47,
  "FIPS_CTY" : 179,
  "LATITUDE" : 36.292787,
  "LONGITUDE" : -82.568171,
  "ADDR_TYP" : 1,
  "MSA" : 3660,
  "CBSA" : 27740,
  "ADDR_LINE" : 3,
  "DMA_SUPPR" : "",
  "GEO_MATCH" : 1,
  "CENS_TRACT" : 61900,
  "CENS_BLK_GRP" : 1,
  "CENS_BLK" : 17,
  "CENS_MED_HOME_VALUE" : 953,
  "CENS_MED_HH_INCOME" : 304,
  "CRA" : "",
  "Z4_TYP" : "S",
  "DSF_IND" : 1,
  "DPD_IND" : "N",
  "PHONE_FLAG" : "Y",
  "PHONE" : NumberLong("4237730233"),
  "TIME_ZN" : "E",
  "GENDER" : "M",
  "NEW_TO_BLD" : "",
  "SOURCES" : 19,
  "BASE_VER_DT" : 20101,
  "COMP_ID" : NumberLong("3769001836"),
  "IND_ID" : 1,
  "INF_HH_RANK" : 1,
  "HOME_OWNR_SRC" : "V",
  "DOB_YR" : 1975,
  "DOB_MON" : 7,
  "DOB_DAY" : 10,
  "EXACT_AGE" : 39,
  "AGE" : 39,
  "HH_INCOME" : "D",
  "NET_WORTH" : "G",
  "CREDIT_LINES" : "",
  "CREDIT_RANGE_NEW" : "",
  "EDUC" : "A",
  "OCC_OCCUP" : "E",
  "OCC_OCCUP_DET" : "",
  "OCC_BUSN_OWNR" : "",
  "NUM_KIDS" : 3,
  "PRES_KIDS" : "Y",
  "KIDS_M_0_2" : "",
  "KIDS_F_0_2" : "",
  "KIDS_U_0_2" : "Y",
  "KIDS_M_3_5" : "",
  "KIDS_F_3_5" : "",
  "KIDS_U_3_5" : "Y",
  "KIDS_M_6_10" : "",
  "KIDS_F_6_10" : "",
  "KIDS_U_6_10" : "",
  "KIDS_M_11_15" : "",
  "KIDS_F_11_15" : "",
  "KIDS_U_11_15" : "",
  "KIDS_M_16_17" : "",
  "KIDS_F_16_17" : "",
  "KIDS_U_16_17" : "Y",
  "HH_MARITAL_STAT" : "S",
  "HOME_OWNR" : "O",
  "LOR" : 7,
  "DWELL_TYP" : "S",
  "NUM_ADULTS" : 1,
  "HH_SIZE" : 2,
  "HOME_MKT_VALUE" : "F",
  "GENS" : 2,
  "ADULTS_M_18_24" : "",
  "ADULTS_F_18_24" : "",
  "ADULTS_U_18_24" : "",
  "ADULTS_M_25_34" : "",
  "ADULTS_F_25_34" : "",
  "ADULTS_U_25_34" : "",
  "ADULTS_M_35_44" : "Y",
  "ADULTS_F_35_44" : "",
  "ADULTS_U_35_44" : "",
  "ADULTS_M_45_54" : "",
  "ADULTS_F_45_54" : "",
  "ADULTS_U_45_54" : "",
  "ADULTS_M_55_64" : "",
  "ADULTS_F_55_64" : "",
  "ADULTS_U_55_64" : "",
  "ADULTS_M_65_74" : "",
  "ADULTS_F_65_74" : "",
  "ADULTS_U_65_74" : "",
  "ADULTS_M_75PLUS" : "",
  "ADULTS_F_75PLUS" : "",
  "ADULTS_U_75PLUS" : ""
  }
  </code></pre>
  
  <p>I have around 200000000 documents in the collection. </p>
  
  <p>Following is my angular controller</p>
  
  <pre><code>$http.post('/api/allsearch',
          {
              "everything":$scope.filterSearch ,
              "searchid":$routeParams.id, 
              "filterId": $scope.filterId, 
              "zipgroup" : $scope.zipgroup
          },{timeout:600000})
          .success(function(data){
  
              for(var i=0; i&lt;data.zips.length;i++){
                  oneset={
                      "zip": data.zips[i],
                      "state": data.states[i],
                      "city": data.cities[i],
                      "distance": data.distances[i],
                      "count": data.counted_results[i]
                  };
                  $scope.totalCount+=data.counted_results[i];
                  $scope.results.push(oneset);
              }
              angular.forEach($scope.results, function (result) {
                      result.distance = parseFloat(result.distance);
                  });
              $rootScope.processing=false;
              $scope.filterlinkdisplay=true;
      }); 
  </code></pre>
  
  <p>And these are my node api functions related to the request.</p>
  
  <pre><code>    api.route('/allsearch')
      .post(function(req,res){
          var filters=req.body.everything;
          var filterid=req.body.filterId;
          var searchid=req.body.searchid;
          var zipgroup=req.body.zipgroup;
          var myObject = new Array();
  
  
          Searchradius.findOne({"searchid" : searchid, user: req.decoded.id}).exec(function(err, docs) {
  
              var asyncTasks = [];
  
              // Loop through some items
              zipgroup.forEach(function(item){
                // We don't actually execute the async action here
                // We add a function containing it to an array of "tasks"
                asyncTasks.push(function(callback){
                  // Call an async function, often a save() to DB
                  searchingalgo(item, filters, filterid,  function(pers){
                          myObject[item] = pers;
                        // Async call is done, alert via callback
                    callback();
                  });
                });
              });
  
  
              Async.parallel(asyncTasks, function(){
                  //console.log(myObject);
                  Searchradius.update({ _id: searchid }, { $set: { ucounteds: myObject , uzips: zipgroup }}, function(err, result){
                      if(err) {
                          res.send(err);
                          return;
                      }
                      var fields = ['city', 'state', 'zip','distance', 'count'];
                      var myresults = [];
                      var tc=0;
                      var newMyobj= new Array();
                      co=0;
                      zipgroup.forEach(function(item){
                          tc+=myObject[item];
                          //myresults.push(jobj);
                      });
  
                      for(i=0;i&lt;zipgroup.length;i++){
                          newMyobj[i]=myObject[zipgroup[i]];
                      }
                      console.log(tc);
                      Searchfilter.update({ _id: filterid }, { $set: { counted_results: tc }}, function(err, resultupdate){
                      });
  
  
                      res.json({ 
                      success: true,
                      zips: zipgroup,
                      states: docs.states,
                      cities: docs.cities,
                      distances: docs.distances,
                      counted_results : newMyobj,
                      });
  
                  });     //update searchradius
              });         //getdata function 
          });             //searchradius findone
      });
  
  
  function searchingalgo(zip, filerobject, filterid, callback){
  
   var query = Consumer1s.count({ "ZIP": zip});
   if(filerobject.ac){
          var ac = Object.keys(filerobject.ac);
          var adultagecriteria= [];
          ac.forEach(function(entry) {
              var prop={};
              prop[entry]="Y";
              adultagecriteria.push(prop);
          });
          query.or(adultagecriteria);
   }
  
   if(filerobject.child){
          var child = Object.keys(filerobject.child);
          var children= [];
          child.forEach(function(entry) {
              var prop={};
              prop[entry]="Y";
              children.push(prop);
          });
          query.or(children);
    }
  
    if(filerobject.HH_INCOME){
      query.where('HH_INCOME', filerobject.HH_INCOME);
    }
  
    if(filerobject.NET_WORTH){
      query.where('NET_WORTH', filerobject.NET_WORTH);
    }
  
    if(filerobject.GENDER){
     query.where('GENDER', filerobject.GENDER);
    }
  
    query.exec(function(err,dc){
      if(filterid){ 
      Searchfilter.update({ _id: filterid }, { $set: { filters: filerobject }}, function(err, result){
              if(err) {
                  console.log(err);
                  return;
              }       
          //  console.log(result);
                  //result= dc;
                  callback(dc);
              });
     }else{
          callback(dc);
      }
    });
   }
  </code></pre>
  
  <p>zipgroup is of the type</p>
  
  <pre><code>[37663, 37664, 37669, 37671, 37660, 37669, 37667, 37668, 37666, 37665, 37662, 37661 ]
  </code></pre>
  
  <p>Basically the concept is counting the number of records based on the ZIP or ADULTS_*info and basically all the fields in the document.</p>
  
  <p>I have done the indexing on ZIP. It helps speeding when there are no filters but if the filters(usually like) include</p>
  
  <pre><code>{ 
  GENDER: '',
  HH_INCOME: 'A',
  OCC_OCCUP: 'B',
  ac: 
  { ADULTS_F_18_24: true,
    ADULTS_F_24_35: true,
    ADULTS_M_18_24: true,
    ADULTS_M_24_35: true },
  child: { KIDS_F_0_2: true, KIDS_F_6_10: true }
   }
  </code></pre>
  
  <p>The query becomes too slow and the node app crashes. 
  1. I need ways to increase the query time of mongodb (apart from indexing). 
  2. I need node app to start again automatically when it has crashed. 
  3. Any other suggestions are welcome.</p>
  
  <p>Also I tried increasing the old_max_space_size to 2000000 but that does not work.. </p>