title: Dumping DOM using chrome headless and GO
body: <h3>What I am trying to do :</h3>
  
  <p>Dump the dom after it is rendered using <strong>chrome headless</strong> and <strong>go</strong>.</p>
  
  <p>I saw over the code of <a href="https://github.com/michenriksen/aquatone/blob/master/agents/url_screenshotter.go#L100" rel="nofollow noreferrer">aqautone</a> to get an idea as to how to use chrome headless for my needs.</p>
  
  <h3>Problem</h3>
  
  <p>The problem is if I add many urls it just time outs printing <code>Dumping dom timed out</code> and currently when there's only two urls it prints : </p>
  
  <pre><code>&lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  &lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  &lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  &lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  Checking  bingBot
  1.73 URL : https://www.facebook.com
  &lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  Checking  yahooBot
  1.75 URL : https://www.facebook.com
  Checking  bingBot
  1.74 URL : https://www.google.com
  Checking  googleBot
  1.75 URL : https://www.google.com
  Checking  googleBot
  1.76 URL : https://www.facebook.com
  &lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;
  Checking  yahooBot
  1.76 URL : https://www.google.com
  Total time taken 1.78s elapsed
  </code></pre>
  
  <p>Here's my go code : <a href="https://play.golang.org/p/QZGlztvzlPY" rel="nofollow noreferrer">Playground</a>
  ( However since there's no chrome exec there so it is no point)</p>
  
  <p>In case you don't want to go to the Playground :</p>
  
  <pre><code>package main
  
  import (
      "context"
      "fmt"
      "io"
      "log"
      "os"
      "os/exec"
      "sync"
      "time"
      //"github.com/remeh/sizedwaitgroup"
  )
  
  // Taken from https://deviceatlas.com/blog/list-of-user-agent-strings
  var (
      useragentstrings = map[string]string{
          "googleBot": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
          "bingBot":   "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
          "yahooBot":  "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
      }
      chromePath = "/root/tools/chromium-latest-linux/latest/chrome"
  )
  
  func fetchUsingChrome(url string, nameOfAgent string, useragent string, ch chan&lt;- string, wg *sync.WaitGroup) {
      // func fetchUsingChrome(url string, nameOfAgent string, useragent string, ch chan&lt;- string, wg *sizedwaitgroup.SizedWaitGroup) {
      start := time.Now()
      defer wg.Done()
      var chromeArguments = []string{
          "--headless", "--disable-gpu", "--hide-scrollbars", "--mute-audio", "--disable-notifications",
          "--disable-crash-reporter",
          "--ignore-certificate-errors",
          "--dump-dom",
          // "--screenshot=" + url + ".png",
          "--user-agent=" + useragent,
      }
      if os.Geteuid() == 0 {
          chromeArguments = append(chromeArguments, "--no-sandbox")
      }
      ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
      defer cancel()
  
      cmd := exec.CommandContext(ctx, chromePath, chromeArguments...)
      stdout, _ := cmd.StdoutPipe()
      if err := cmd.Start(); err != nil {
          fmt.Printf("[%s]Dumping dom failed Error: %v\n", err)
          return
      }
      // buf := new(bytes.Buffer)
  
      // if err := buf.ReadFrom(stdout); err != nil {
      // fmt.Println(err)
      // return
      // }
      // fmt.Println(buf.String())
      if _, err := io.Copy(os.Stdout, stdout); err != nil {
          log.Fatal(err)
      }
  
      if err := cmd.Wait(); err != nil {
          if ctx.Err() == context.DeadlineExceeded {
              fmt.Printf("Dumping dom timed out : %s\n", url)
              return
          }
          fmt.Printf("[%s]Dumping dom failed Error: %v\n", err)
          return
      }
      secs := time.Since(start).Seconds()
      fmt.Println("Checking ", nameOfAgent)
      ch &lt;- fmt.Sprintf("%.2f URL : %s", secs, url)
  }
  
  func main() {
      var wg sync.WaitGroup
      output := []string{
          "https://www.facebook.com",
          "https://www.google.com",
      }
      start := time.Now()
      ch := make(chan string)
      //swg := sizedwaitgroup.New(10)
      for _, url := range output {
          for k, v := range useragentstrings {
              wg.Add(1)
              //swg.Add()
              go fetchUsingChrome(url, k, v, ch, &amp;wg)
              // go fetchUsingChrome(url, k, v, ch, &amp;swg)
          }
      }
  
      go func() {
          wg.Wait()
          // swg.Wait()
          close(ch)
      }()
      for val := range ch {
          fmt.Println(val)
      }
      fmt.Printf("Total time taken %.2fs elapsed\n", time.Since(start).Seconds())
  }
  
  </code></pre>
  
  <p><strong>P.S.</strong> : I've deliberately left some commented code, to give an insight as to what all I've tried and failed.</p>
  
  <p>Thanks<br>
  Temporarya<br>
  ( A golang noobie )</p>