src/events/crawler/tasks/scrapeData/runScraper.js
/*
  Run the correct scraper for this location
*/
Run the correct scraper for this location
Run the correct scraper for this location
function runScraper(location) {
  const rejectUnauthorized = location.certValidation === false;
  if (rejectUnauthorized) {
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
  }
  if (typeof location.scraper === 'function') {
    return location.scraper();
  }
  if (rejectUnauthorized) {
    delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
  }
  if (typeof location.scraper === 'object') {
    const targetDate = process.env.SCRAPE_DATE || datetime.getDate();
    let scraperToUse = null;
    for (const [date, scraper] of Object.entries(location.scraper)) {
      if (datetime.dateIsBeforeOrEqualTo(date, targetDate)) {
        scraperToUse = scraper;
      }
    }
    if (scraperToUse === null) {
      throw new Error(
        `Could not find scraper for ${geography.getName(location)} at ${
          process.env.SCRAPE_DATE
        }, only have: ${Object.keys(location.scraper).join(', ')}`
      );
    }
    return scraperToUse.call(location);
  }
  throw new Error('Why on earth is the scraper for %s a %s?', geography.getName(location), typeof scraper);
}
function runScraper ( location ) { const rejectUnauthorized = location . certValidation === false ; if ( rejectUnauthorized ) { process . env . NODE_TLS_REJECT_UNAUTHORIZED = '0' ; } if ( typeof location . scraper === 'function' ) { return location . scraper ( ) ; } if ( rejectUnauthorized ) { delete process . env . NODE_TLS_REJECT_UNAUTHORIZED ; } if ( typeof location . scraper === 'object' ) { const targetDate = process . env . SCRAPE_DATE || datetime . getDate ( ) ; let scraperToUse = null ; for ( const [ date , scraper ] of Object . entries ( location . scraper ) ) { if ( datetime . dateIsBeforeOrEqualTo ( date , targetDate ) ) { scraperToUse = scraper ; } } if ( scraperToUse === null ) { throw new Error ( ` ${ geography . getName ( location ) } ${ process . env . SCRAPE_DATE } ${ Object . keys ( location . scraper ) . join ( ', ' ) } ` ) ; } return scraperToUse . call ( location ) ; } throw new Error ( 'Why on earth is the scraper for %s a %s?' , geography . getName ( location ) , typeof scraper ) ; }
Run the correct scraper for this location
Run the correct scraper for this location
// Important: this prevents SSL from failing // Find the closest date