src/similarweb.js
/**
 * Crawl list item traffic from similarweb and modify crawllist directly.
 * @param {object} driver Selenium web driver.
 * @param {object} crawlList An object containing crawler status and list data.
 * @param {func} getData function that returns data segment in the crawlList data structure
 * @param {string} fileName output fileName
 * @param {string} timeIntervalString time
 */
Crawl list item traffic from similarweb and modify crawllist directly.
@param {object} driver Selenium web driver.
@param {object} crawlList An object containing crawler status and list data.
@param {func} getData function that returns data segment in the crawlList data structure
@param {string} fileName output fileName
@param {string} timeIntervalString time
Crawl list item traffic from similarweb and modify crawllist directly . @param { object } driver Selenium web driver . @param { object } crawlList An object containing crawler status and list data . @param { func } getData function that returns data segment in the crawlList data structure @param { string } fileName output fileName @param { string } timeIntervalString time
async function crawlListTraffic(driver, crawlList, getData, fileName, timeIntervalString) {
  console.log('Crawl traffic from similar web...');
  if (timeIntervalString) timeInterval = timeIntervalString;
  let item;
  let list = getData(crawlList);
  await signInSimilarWeb(driver, 'google');
  for (let i=0; i<list.length; i++) {
    console.log(i);
    if (list[i].hasOwnProperty('traffic') && list[i].traffic.success) continue;
    item = list[i];
    console.log('Crawling ' + item.name + ' traffic...');
    if (getDomainName(item.url) === 'dappradar.com') {
      await driver.get(item.url);
      await sleep(3000);
      item.url = await driver.getCurrentUrl();
      await driver.get('https://pro.similarweb.com/#/home/modules');
      await driver.wait(until.titleIs('SimilarWeb Home'), 100000);
    }
    item.traffic = await getTraffic(driver, getDomainName(item.url));
    if (i%5 == 4) await writeFile(fileName, JSON.stringify(crawlList, null, 2));
  }
  await writeFile(fileName, JSON.stringify(crawlList, null, 2));
}
async function crawlListTraffic ( driver , crawlList , getData , fileName , timeIntervalString ) { console . log ( 'Crawl traffic from similar web...' ) ; if ( timeIntervalString ) timeInterval = timeIntervalString ; let item ; let list = getData ( crawlList ) ; await signInSimilarWeb ( driver , 'google' ) ; for ( let i = 0 ; i < list . length ; i ++ ) { console . log ( i ) ; if ( list [ i ] . hasOwnProperty ( 'traffic' ) && list [ i ] . traffic . success ) continue ; item = list [ i ] ; console . log ( 'Crawling ' + item . name + ' traffic...' ) ; if ( getDomainName ( item . url ) === 'dappradar.com' ) { await driver . get ( item . url ) ; await sleep ( 3000 ) ; item . url = await driver . getCurrentUrl ( ) ; await driver . get ( 'https://pro.similarweb.com/#/home/modules' ) ; await driver . wait ( until . titleIs ( 'SimilarWeb Home' ) , 100000 ) ; } item . traffic = await getTraffic ( driver , getDomainName ( item . url ) ) ; if ( i % 5 == 4 ) await writeFile ( fileName , JSON . stringify ( crawlList , null , 2 ) ) ; } await writeFile ( fileName , JSON . stringify ( crawlList , null , 2 ) ) ; }
Crawl list item traffic from similarweb and modify crawllist directly.
Crawl list item traffic from similarweb and modify crawllist directly .