app.js
/**
 * scan_subreddits 
 * 
 * Main worker function - establishes statements and intervals for Reddit API
 * calls.
 */


Main worker function - establishes statements and intervals for Reddit API
calls.
Main worker function - establishes statements and intervals for Reddit API calls .
function scan_subreddits() {
  C.log('Beginning scan');
  var fields = ['subscribers', 'accounts_active', 'public_traffic', 'created_utc', 
                'quarantine', 'accounts_active_is_fuzzed'];
  var insert = DB.prepare(`INSERT INTO sub_info 
                                (sub_id, timestamp, ${fields.join(', ')}) 
                          VALUES (?, ?${Array(fields.length+1).join(', ?')});`);
  SUBSCRIPTION_TIMER = setInterval(() => {
    if (SUB_INDEX == -1) {
      C.log('Database not ready. Waiting...');
      return;
    }
    let row = SUB_LIST[SUB_INDEX];
    SUB_INDEX = (SUB_INDEX + 1) % SUB_LIST.length;
    if (SKIPPED_SUBS.indexOf(row.name) > -1) {
      return;
    }
    try { 
      reddit_api.get({'path': `/r/${row.name}/about`}, (err, response) => {
        if (err) {
          if (err === reddit_api.ERR_JSON_PARSE) {
            var f = C.save_object(response);
            C.log("ERROR [JSON]: Could not parse response. Data saved to ", f);
          } else if (err === reddit_api.ERR_NOT_READY 
                      || err === reddit_api.ERR_SERVER) {
          } else {
            C.log("ERROR [API]:", err);
          }
          return;
        }
        var values = [row.id, Date.now()];
        if ('error' in response.content && response.content.error == 404) {
          C.log(`Sub ${row.name} returned 404, setting to inactive`);
          DB.run(`UPDATE subs SET is_active = 0 WHERE id = ${row.id}`);
          SKIPPED_SUBS.push(row.name);
          return;
        }
        for (var i in fields) {
          try {
            values.push(response.content.data[fields[i]]);
          } catch (err) {
            var filename = C.save_object(response);
            throw new Error (
              `Missing field ${fields[i]}. Object saved to ${filename}`);
          }
        }
        insert.run(values);
      });
    } catch(exc) {
      C.log("ERROR [EXCEPTION]: ", exc)
    }
  }, SUBSCRIPTION_INTERVAL);
}
function scan_subreddits ( ) { C . log ( 'Beginning scan' ) ; var fields = [ 'subscribers' , 'accounts_active' , 'public_traffic' , 'created_utc' , 'quarantine' , 'accounts_active_is_fuzzed' ] ; var insert = DB . prepare ( ` ${ fields . join ( ', ' ) } ${ Array ( fields . length + 1 ) . join ( ', ?' ) } ` ) ; SUBSCRIPTION_TIMER = setInterval ( ( ) => { if ( SUB_INDEX == - 1 ) { C . log ( 'Database not ready. Waiting...' ) ; return ; } let row = SUB_LIST [ SUB_INDEX ] ; SUB_INDEX = ( SUB_INDEX + 1 ) % SUB_LIST . length ; if ( SKIPPED_SUBS . indexOf ( row . name ) > - 1 ) { return ; } try { reddit_api . get ( { 'path' : ` ${ row . name } ` } , ( err , response ) => { if ( err ) { if ( err === reddit_api . ERR_JSON_PARSE ) { var f = C . save_object ( response ) ; C . log ( "ERROR [JSON]: Could not parse response. Data saved to " , f ) ; } else if ( err === reddit_api . ERR_NOT_READY || err === reddit_api . ERR_SERVER ) { } else { C . log ( "ERROR [API]:" , err ) ; } return ; } var values = [ row . id , Date . now ( ) ] ; if ( 'error' in response . content && response . content . error == 404 ) { C . log ( ` ${ row . name } ` ) ; DB . run ( ` ${ row . id } ` ) ; SKIPPED_SUBS . push ( row . name ) ; return ; } for ( var i in fields ) { try { values . push ( response . content . data [ fields [ i ] ] ) ; } catch ( err ) { var filename = C . save_object ( response ) ; throw new Error ( ` ${ fields [ i ] } ${ filename } ` ) ; } } insert . run ( values ) ; } ) ; } catch ( exc ) { C . log ( "ERROR [EXCEPTION]: " , exc ) } } , SUBSCRIPTION_INTERVAL ) ; }
scan_subreddits
scan_subreddits
// not happy about the scope of this, but it needs to be bulletproof. // pass // banned subs