title: How to use Puppeteer to scrape a reddit page?
body: <p>I am trying to learn to use Puppeteer to scrape a reddit page. The new reddit has dynamically added content and infinite scrolling. I am getting pretty inconsistent results from code and having a hard time debugging and figuring out how to make this work.</p>
  
  <p>Main server.js file, not much going on here.</p>
  
  <pre><code>'use strict';
  
  const express = require('express');
  const cors = require('cors');
  const app = express();
  const cheerio = require('./redditScraper');
  
  app.use(express.json());
  app.use(
      cors({
          origin: ['http://localhost:3000']
      })
  );
  
  app.get('/', (req, res) =&gt; {
      let { dynamicScraper } = cheerio;
  
      dynamicScraper()
          .then(html =&gt; {
              console.log('data was sent');
              res.json(html);
          })
          .catch(err =&gt; {
              console.log(err);
          });
  });
  
  app.listen(process.env.PORT || 8080, () =&gt; {
      console.log(`Your app is listening on port ${process.env.PORT || 8080}`);
  });
  </code></pre>
  
  <p>File with the scraper stuff</p>
  
  <pre><code>'use strict';
  
  const rp = require('request-promise');
  const $ = require('cheerio');
  const puppeteer = require('puppeteer');
  const url2 = 'https://www.reddit.com/r/GameDeals/';
  
  
  
  const cheerio = {
      dynamicScraper: function() {
         return puppeteer
              .launch()
              .then(browser =&gt; {
                  return browser.newPage();
              })
              .then(page =&gt; {
                  return page.goto(url2)
                      .then(() =&gt; {
                          //attempting to scroll through page to cause loading in more elements, doesn't seem to work
                          page.evaluate(() =&gt; {
                              window.scrollBy(0, window.innerHeight);
                          })
                          return page.content()
                      });
              })
              .then(html =&gt; {
                  //should log the the first post's a tag's href value
                  console.log($('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a', html).attr('href'));
  
                  const urls = [];
  
                  //should be the total number of a tag's across all posts
                  const numLinks = $('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a', html).attr('href').length;
  
                  const links = $('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a', html);
  
                  //was using numLinks as increment counter but was getting undefined, as the pages only seems to load inconsistent between 10-20 elements
                  for (let i=0; i&lt;10; i++) {
                      urls.push(links[i].attribs.href);
                  }
  
                  console.log('start of urls:', urls);
                  console.log('scraped urls:', urls.length);
                  console.log('intended number of urls to be scraped:', numLinks);
                  // console.log($('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a', html).length);
              })
              .catch(err =&gt; console.log(err));
      }
  
  }
  
  module.exports = cheerio;
  </code></pre>
  
  <p>The above code currently works, however as you can see from the comments, I have the counter hard coded to 10, which is obviously not the total number of <code>&lt;a href=#&gt;</code> on the page.</p>
  
  <p>Here is the output for the above:</p>
  
  <pre><code>[nodemon] starting `node server.js`
  Your app is listening on port 8080
  https://www.gamebillet.com/garfield-kart
  start of urls: [ 'https://www.gamebillet.com/garfield-kart',
    'https://www.humblebundle.com/store/deep-rock-galactic?hmb_source=humble_home&amp;hmb_medium=product_tile&amp;hmb_campaign=mosaic_section_1_layout_index_9_layout_type_twos_tile_index_1',  'https://www.greenmangaming.com/games/batman-arkham-asylum-game-of-the-year/',
    'https://www.humblebundle.com/store/ftl-faster-than-light',
    'https://www.greenmangaming.com/vip/vip-deals/',
    'https://store.steampowered.com/app/320300/',
    'https://store.steampowered.com/app/356650/Deaths_Gambit/',
    'https://www.chrono.gg/?=Turmoil',
    'https://www.fanatical.com/en/game/slain',
    'https://freebies.indiegala.com/super-destronaut/?dev_id=freebies' ]
  scraped urls: 10
  numLinks: 40
  data was sent
  </code></pre>
  
  <p>Here is the output when the for loop is changed to <code>numlinks</code></p>
  
  <pre><code>for (let i=0; i&lt;numLinks; i++) {
      urls.push(links[i].attribs.href);
  }
  </code></pre>
  
  <pre><code>[nodemon] starting `node server.js`
  Your app is listening on port 8080
  https://www.gamebillet.com/garfield-kart
  TypeError: Cannot read property 'attribs' of undefined
      at puppeteer.launch.then.then.then.html (/file.js:49:40)
      at process._tickCallback (internal/process/next_tick.js:68:7)
  data was sent
  </code></pre>
  
  <p>I hope this isn't too much of a mess to read. I would appreciate any help. Thank you.</p>
  
  <h1>Update/edit:</h1>
  
  <p>I am trying to implement the async way, but I'm not sure how to return the value to be used in the route handler?</p>
  
  <pre><code>    dynamicScraper: function() {
          async function f() {
              const browser = await puppeteer.launch();
              const [page] = await browser.pages();
  
              await page.goto(url2, { waitUntil: 'networkidle0' });
              const links = await page.evaluate(async () =&gt; {
                  const scrollfar = document.body.clientHeight;
                  console.log(scrollfar); //trying to find the height
                  window.scrollBy(0, scrollfar);
                  await new Promise(resolve =&gt; setTimeout(resolve, 5000)); 
                  return [...document.querySelectorAll('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a')]
                      .map((el) =&gt; el.href);
              });
              console.log(links, links.length);
  
              await browser.close();
              //how do I return the value to pass to the route handler?
              return (links);
          };
          return(f());
      }
  </code></pre>
  
  <p>I am getting from the <code>console.log</code>.</p>
  
  <pre><code>Your app is listening on port 8080
  [ 'https://www.nintendo.com/games/detail/celeste-switch',
    'https://store.steampowered.com/app/460790/Bayonetta/',
    'https://www.greenmangaming.com/de/vip/vip-deals/']
  </code></pre>
  
  <p>But the response from server to client is an empty object on browser
  <code>{}</code></p>
  
  <h1>Update/edit 2:</h1>
  
  <p>Nevermind, figured it out needed to handle the promise from the async function.</p>
  
  <pre><code>dynamicScraper().then((output) =&gt; {
          res.json(output);
      });
  </code></pre>