imagescraper.js
// Scrape: Start puppeteer, open URL, parse the DOM, extract images, save images
Start puppeteer, open URL, parse the DOM, extract images, save images
Start puppeteer open URL parse the DOM extract images save images
async function scrape(url, outputDir) {
  console.log('Starting puppeteer');
  const browser = await puppeteer.launch({
    headless: true,
    ignoreDefaultArgs: ['--disable-extensions'],
    args: [
      '--no-sandbox',
      '--no-zygote',
      '--single-process'
    ]
  });
  console.log('Set viewport');
  const page = await browser.newPage();
  await page.setViewport({
    width: 990,
    height: 800
  });
  console.log('Opening URL:', url);
  try {
    await page.goto(url, {
      waitUntil: 'load',
      timeout: 180000
    });
  } catch (e) {
    console.log('Unable to open URL:', url, e);
    await browser.close();
    process.exit(1);
  }
  console.log('URL opened successfully');
  let images = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('img.fp-image'), e => ({ id: e.getAttribute('id'), src: e.getAttribute('src') }))
  })
  await browser.close();
  console.log('Images found:', images.length);
  images.forEach(image => {
    const data = image.src.replace(/^data:image\/\w+;base64,/, '');
    const bitmap = Buffer.from(data, 'base64'); 
    const fileName = image.id.replace('-canvas', '').replace('-png', '') + '.png';
    console.log('Saving:', fileName)
    try {
      fs.writeFileSync(path.join(outputDir, fileName), bitmap);
    }
    catch (e) {
      console.log('Error writing file:', fileName, e);
    }
  });
  console.log('Scraping done');
}
async function scrape ( url , outputDir ) { console . log ( 'Starting puppeteer' ) ; const browser = await puppeteer . launch ( { headless : true , ignoreDefaultArgs : [ '--disable-extensions' ] , args : [ '--no-sandbox' , '--no-zygote' , '--single-process' ] } ) ; console . log ( 'Set viewport' ) ; const page = await browser . newPage ( ) ; await page . setViewport ( { width : 990 , height : 800 } ) ; console . log ( 'Opening URL:' , url ) ; try { await page . goto ( url , { waitUntil : 'load' , timeout : 180000 } ) ; } catch ( e ) { console . log ( 'Unable to open URL:' , url , e ) ; await browser . close ( ) ; process . exit ( 1 ) ; } console . log ( 'URL opened successfully' ) ; let images = await page . evaluate ( ( ) => { return Array . from ( document . querySelectorAll ( 'img.fp-image' ) , e => ( { id : e . getAttribute ( 'id' ) , src : e . getAttribute ( 'src' ) } ) ) } ) await browser . close ( ) ; console . log ( 'Images found:' , images . length ) ; images . forEach ( image => { const data = image . src . replace ( / ^data:image\/\w+;base64, / , '' ) ; const bitmap = Buffer . from ( data , 'base64' ) ; const fileName = image . id . replace ( '-canvas' , '' ) . replace ( '-png' , '' ) + '.png' ; console . log ( 'Saving:' , fileName ) try { fs . writeFileSync ( path . join ( outputDir , fileName ) , bitmap ) ; } catch ( e ) { console . log ( 'Error writing file:' , fileName , e ) ; } } ) ; console . log ( 'Scraping done' ) ; }
Scrape: Start puppeteer, open URL, parse the DOM, extract images, save images
Scrape : Start puppeteer open URL parse the DOM extract images save images
// Launch Puppeteer, take care with options for WSL/2 // For WSL2, found this deep in a github discussion // Load page, set viewport to optimal size for charts // Open the URL, set a loooong timeout - the 30sec default is too short // Evaluate the page, extract all images //.slice(1, 5); // Save all images, base filename on 'id' tag // images.src starts with the following // data:image/png;base64,iVBORw0KGgoAAAANSUhE....... // We need to remove the preamble 'data:image/png;base64,' and just keep the base64 encoded data // convert to Buffer for saving // We're done