title: How can I get all the comments/reviews from a google maps place using puppeteer? (I dont get all of them because page is scrollable)
body: <p>I am trying to scrape comments/reviews from a place I search using Puppeteer. I have 2 problems:</p>
  
  <ol>
  <li>I only get 16 comments/reviews from the current page, when in reality I want ALL the comments/reviews (in this case 62 comments or even more depending on my search) but I think the problem comes from the page being scrollable.</li>
  <li><p>I am getting an error when I scrape reviews that have no comments in google maps saying: </p>
  
  <pre><code>"(node:13184) UnhandledPromiseRejectionWarning: Error: Evaluation failed: TypeError: Cannot read property 'innerText' of null
  at __puppeteer_evaluation_script__:9:38"
  </code></pre>
  
  <p>, and I am not sure how to get rid of that everytime there is a review that has a NULL comment (I have some code almost at the end trying to solve the NULL comments but doesn't work and I tried a few other ways that didn't work either).</p></li>
  </ol>
  
  <p>Below is my code:</p>
  
  <pre><code>const puppeteer = require('puppeteer'); // Require the Package we need...
  
  let scrape = async () =&gt; { // Prepare scrape...
  
      const browser = await puppeteer.launch({args: ['--no-sandbox', '--disabled-setuid-sandbox']}); // Prevent non-needed issues for *NIX
      const page = await browser.newPage(); // Create request for the new page to obtain...
  
      const busqueda = 'Alitas+del+Cadillac+Tumbaco';
      const Url = `https://www.google.com/maps/search/${busqueda}`;
  
      const buscar = '.section-result';
      const click1 = '.widget-pane-link';
      const cajaTexto = '#searchboxinput';
  
      const comentarioLength = 'section-review-text';
      const comentarios = 'div.section-review:nth-child(Index) &gt; div:nth-child(1) &gt; div:nth-child(3) &gt; div:nth-child(2) &gt; div:nth-child(1) &gt; span:nth-child(4)';
  
      console.log(comentarioLength);
  
      //const comentario = 'div.section-review:nth-child(INDEX) &gt; div:nth-child(1) &gt; div:nth-child(3) &gt; div:nth-child(2) &gt; div:nth-child(1) &gt; span:nth-child(4)';
  
      // Replace with your Google Maps URL... Or Test the Microsoft one...
      //await page.goto('https://www.google.com/maps/place/Microsoft/@36.1275216,-115.1728651,17z/data=!3m1!5s0x80c8c416a26be787:0x4392ab27a0ae83e0!4m7!3m6!1s0x80c8c4141f4642c5:0x764c3f951cfc6355!8m2!3d36.1275216!4d-115.1706764!9m1!1b1');
  
      await page.goto(Url); // Define the Maps URL to Scrape...
      await page.waitFor(2*1000); // In case Server has JS needed to be loaded...
  
      await page.click(buscar); //busco caja de texto*/
  
      await page.waitForNavigation();
      await page.waitFor(2*1000);
  
      await page.click(click1);
  
      await page.waitForNavigation();
      await page.waitFor(2*1000);
  
      console.log(page.url());
  
      console.log("3");
  
      await page.evaluate(_ =&gt; { // This is just a test, don't really need this!
  
      });
  
      await page.waitFor(2*1000);
  
      console.log('how many?', (await page.$$('.section-review-text')).length);
  
      //div.section-result:nth-child(1) &gt; div:nth-child(1) &gt; div:nth-child(1) &gt; div:nth-child(1) &gt; div:nth-child(2) &gt; h3:nth-child(1) &gt; span:nth-child(1)
  
      let listLength = await page.evaluate((sel) =&gt; {
          window.scrollBy(0, window.innerHeight);
          return document.getElementsByClassName(sel).length;
      }, comentarioLength);
  
      console.log(listLength);
  
      for (let i = 1; i &lt;= listLength; i++) {
  
          let selectorComentarios = comentarios.replace("Index", i);
  
          const result = await page.evaluate((sel) =&gt; { // Let's create variables and store values...
  
              return document.querySelector(sel).innerText;
  
          }, selectorComentarios);
  
          if(!result){
              continue;
          }
  
          console.log(i+result);
  
      }
  
      /*await page.evaluate(_ =&gt; {
          window.scrollBy(0, window.innerHeight)
      })*/
  
      browser.close(); // Close the Browser...
      return result; // Return the results with the Review...
  };
  
  scrape().then((value) =&gt; { // Scrape and output the results...
  
  
  console.log(value); // Yay, output the Results...
  });
  </code></pre>