title: Speed up web scraping using multiplie Rselenium browsers
body: <p>I am using Rselenium to scrape following website:
  <a href="http://plovila.pomorstvo.hr/" rel="nofollow noreferrer">http://plovila.pomorstvo.hr/</a></p>
  <p>Every time I have to enter 'NIB' field, execute and scrape all data.
  I am using Sys.time() function several time so my code is slow (cca 12 seconds for one NIB). I need to scrape around 200.000 NIB numbers which gives 30 days of scraping.</p>
  <p>I am interested if I can open multiple browsers locally or somehow in the cloud and make my scraping script faster.</p>
  <p>Is it possible to use parallel computing to overcome this issue?<br />
  Do you have any suggestions?</p>
  <p>EDIT:<br />
  I am adding the code:</p>
  <pre><code>library(XML)
  library(RCurl)
  library(RSelenium)
  library(png)
  library(imager)
  library(RMySQL)
  library(htmltab)
  library(jsonlite)
  library(rvest)
  
  # function for waiting instead Sys.sleep()
  waitLoad &lt;- function (xpath_check = &quot;//input[@id = 'ctl00_Content_FormContent_uiIspisGrid_ctl00__0']/td[2]&quot;,
                         iterations = 5){
    counter &lt;- 0
    chk &lt;- FALSE
    while(!chk &amp; counter &lt;= iterations){
      wait &lt;- tryCatch(
        remDr$findElement(using = &quot;xpath&quot;,
                          xpath_check)$getElementText(),
        # remDr$findElement(using = &quot;xpath&quot;, &quot;//input[@id = 'ctl00_Content_FormContent_Img1']&quot;)$clearElement(),
        error = function(e) print(paste0(&quot;Trazi dalje&quot;))
      )
      if(wait == &quot;Trazi dalje&quot; ){
        Sys.sleep(1L)
        counter &lt;- sum(counter, 1)
      }else{
        chk &lt;- TRUE
      }
    }
  }
  
  # Start Selenium Server
  # docker run -d -p 4445:4444 selenium/standalone-chrome:3.5.0
  remDr &lt;- remoteDriver(remoteServerAddr = &quot;192.168.99.100&quot;, port = 4445L, browserName = &quot;chrome&quot;)
  remDr$open()
  
  # Simulate browser session and fill out form
  remDr$navigate(&quot;http://plovila.pomorstvo.hr/&quot;)
  remDr$findElement(using = &quot;xpath&quot;, &quot;//select[@id = 'ctl00_Content_FormContent_uiTipObjektaDropDown']/option[@value = '1']&quot;)$clickElement()
  remDr$screenshot(display = TRUE)
  
  # Scrap !
  df &lt;- list()
  Porivni_uredjaji &lt;- list()
  Clanovi_posade &lt;- list()
  Vlasnici &lt;- list()
  Korisnici &lt;- list()
  df_2 &lt;- list()
  Tereti &lt;- list()
  pocetak &lt;- 100000
  kraj &lt;- 100003
  system.time(
  for (i in pocetak:kraj){
    remDr$findElement(using = &quot;xpath&quot;, &quot;//input[@id = 'ctl00_Content_FormContent_uiNibTextBox']&quot;)$clearElement()
    Sys.sleep(1L)
    remDr$findElement(using = &quot;xpath&quot;, 
                      &quot;//input[@id = 'ctl00_Content_FormContent_uiNibTextBox']&quot;)$sendKeysToElement(list(as.character(i), 
                                                                                                           key = &quot;enter&quot;))
    waitLoad()
    remDr$screenshot(display = TRUE)
    doc &lt;- htmlParse(remDr$getPageSource()[[1]])
    Sys.sleep(1L)
    Ime &lt;- xpathSApply(doc = doc, path = &quot;//*[@id='ctl00_Content_FormContent_uiIspisGrid_ctl00__0']/td[1]&quot;, fun = xmlValue)
    Oznaka &lt;- xpathSApply(doc = doc, path = &quot;//*[@id='ctl00_Content_FormContent_uiIspisGrid_ctl00__0']/td[2]&quot;, fun = xmlValue)
    NIB &lt;- xpathSApply(doc = doc, path = &quot;//*[@id='ctl00_Content_FormContent_uiIspisGrid_ctl00__0']/td[3]&quot;, fun = xmlValue)
    Vlasnik &lt;- xpathSApply(doc = doc, path = &quot;//*[@id='ctl00_Content_FormContent_uiIspisGrid_ctl00__0']/td[4]&quot;, fun = xmlValue)
    LK_LI &lt;- xpathSApply(doc = doc, path = &quot;/html/body/form/div[4]/div[1]/div[3]/table/tbody/tr/td[5]&quot;, fun = xmlValue)
    br1 &lt;- xpathSApply(doc = doc, path = &quot;/html/body/form/div[4]/div[1]/div[3]/table/tbody/tr/td[6]&quot;, fun = xmlValue)
    br2 &lt;- xpathSApply(doc = doc, path = &quot;/html/body/form/div[4]/div[1]/div[3]/table/tbody/tr/td[7]&quot;, fun = xmlValue)
    x &lt;- i-pocetak + 1
    if (length(NIB)==0){
      Pozivni_znak &lt;- NA
      df[[x]] &lt;- cbind(Ime, Oznaka, NIB, Vlasnik, LK_LI, br1, br2, Pozivni_znak)
      df[[x]] &lt;- as.data.frame(df[[x]], stringsAsFactors = FALSE)
    }else{
      remDr$findElement(using = &quot;xpath&quot;, &quot;//input[@title = 'Detalji']&quot;)$clickElement()
      waitLoad(&quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiNamjenaText']&quot;, 5)
      doc &lt;- htmlParse(remDr$getPageSource()[[1]], encoding = &quot;UTF-8&quot;)
      Sys.sleep(1L)
      list_a &lt;- xpathSApply(doc = doc, path = &quot;/html/body/form/div[4]/div[1]/div[3]/fieldset/h3[1]&quot;, fun = xmlValue)
      if (length(list_a) &gt;= 1){
        
        Namjena &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiNamjenaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Namjena, '&quot;', &quot;]&quot;)
        Namjena &lt;- fromJSON(json)
        Namjena &lt;- as.data.frame(Namjena, stringsAsFactors = FALSE)
        colnames(Namjena) &lt;- &quot;Namjena&quot;
        Vrsta_plovila &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiVrstaPlovilaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Vrsta_plovila, '&quot;', &quot;]&quot;)
        Vrsta_plovila &lt;- fromJSON(json)
        Vrsta_plovila &lt;- as.data.frame(Vrsta_plovila, stringsAsFactors = FALSE)
        colnames(Vrsta_plovila) &lt;- &quot;Vrsta_plovila&quot;
        Model_plovila &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiModelPlovilaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Model_plovila, '&quot;', &quot;]&quot;)
        Model_plovila &lt;- fromJSON(json)
        Model_plovila &lt;- as.data.frame(Model_plovila, stringsAsFactors = FALSE)
        colnames(Model_plovila) &lt;- &quot;Model_plovila&quot;
        Duljina_trupa &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiDuljinaTrupaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Duljina_trupa, '&quot;', &quot;]&quot;)
        Duljina_trupa &lt;- fromJSON(json)
        Duljina_trupa &lt;- as.data.frame(Duljina_trupa, stringsAsFactors = FALSE)
        colnames(Duljina_trupa) &lt;- &quot;Duljina_trupa&quot;
        Sirina_trupa &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiSirinaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Sirina_trupa, '&quot;', &quot;]&quot;)
        Sirina_trupa &lt;- fromJSON(json)
        Sirina_trupa &lt;- as.data.frame(Sirina_trupa, stringsAsFactors = FALSE)
        colnames(Sirina_trupa) &lt;- &quot;Sirina_trupa&quot;
        Visina_trupa &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiVisinaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Visina_trupa, '&quot;', &quot;]&quot;)
        Visina_trupa &lt;- fromJSON(json)
        Visina_trupa &lt;- as.data.frame(Visina_trupa, stringsAsFactors = FALSE)
        colnames(Visina_trupa) &lt;- &quot;Visina_trupa&quot;
        Gaz &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiGazText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Gaz, '&quot;', &quot;]&quot;)
        Gaz &lt;- fromJSON(json)
        Gaz &lt;- as.data.frame(Gaz, stringsAsFactors = FALSE)
        colnames(Gaz) &lt;- &quot;Gaz&quot;
        Nosivost &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiNosivostText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Nosivost, '&quot;', &quot;]&quot;)
        Nosivost &lt;- fromJSON(json)
        Nosivost &lt;- as.data.frame(Nosivost, stringsAsFactors = FALSE)
        colnames(Nosivost) &lt;- &quot;Nosivost&quot;
        GT &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiGtText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', GT, '&quot;', &quot;]&quot;)
        GT &lt;- fromJSON(json)
        GT &lt;- as.data.frame(GT, stringsAsFactors = FALSE)
        colnames(GT) &lt;- &quot;GT&quot;
        Snaga_motora &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiUkupnaSnagaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Snaga_motora, '&quot;', &quot;]&quot;)
        Snaga_motora &lt;- fromJSON(json)
        Snaga_motora &lt;- as.data.frame(Snaga_motora, stringsAsFactors = FALSE)
        colnames(Snaga_motora) &lt;- &quot;Snaga_motora&quot;
        Brodogradiliste &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiBrodogradilisteText']/@value&quot;)
        Brodogradiliste &lt;- gsub(&quot;\&quot;&quot;, &quot;'&quot;, Brodogradiliste)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Brodogradiliste, '&quot;', &quot;]&quot;)
        Brodogradiliste &lt;- fromJSON(json)
        Brodogradiliste &lt;- as.data.frame(Brodogradiliste, stringsAsFactors = FALSE)
        Encoding(Brodogradiliste[,c(1)]) &lt;- &quot;UTF-8&quot;
        colnames(Brodogradiliste) &lt;- &quot;Brodogradiliste&quot;
        Godina_gradnje &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiGodGradnjeText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Godina_gradnje, '&quot;', &quot;]&quot;)
        Godina_gradnje &lt;- fromJSON(json)
        Godina_gradnje &lt;- as.data.frame(Godina_gradnje, stringsAsFactors = FALSE)
        colnames(Godina_gradnje) &lt;- &quot;Godina_gradnje&quot;
        Materijal &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiMaterijalGradnjeText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Materijal, '&quot;', &quot;]&quot;)
        Materijal &lt;- fromJSON(json)
        Materijal &lt;- as.data.frame(Materijal, stringsAsFactors = FALSE)
        colnames(Materijal) &lt;- &quot;Materijal&quot;
        Najveci_broj_osoba &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiMaxBrojOsobaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Najveci_broj_osoba, '&quot;', &quot;]&quot;)
        Najveci_broj_osoba &lt;- fromJSON(json)
        Najveci_broj_osoba &lt;- as.data.frame(Najveci_broj_osoba, stringsAsFactors = FALSE)
        colnames(Najveci_broj_osoba) &lt;- &quot;Najveci_broj_osoba&quot;
        Najveci_broj_putnika &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiMaxBrojPutnikaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Najveci_broj_putnika, '&quot;', &quot;]&quot;)
        Najveci_broj_putnika &lt;- fromJSON(json)
        Najveci_broj_putnika &lt;- as.data.frame(Najveci_broj_putnika, stringsAsFactors = FALSE)
        colnames(Najveci_broj_putnika) &lt;- &quot;Najveci_broj_putnika&quot;
        Najmanji_broj_posade &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiMinBrojPosade']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Najmanji_broj_posade, '&quot;', &quot;]&quot;)
        Najmanji_broj_posade &lt;- fromJSON(json)
        Najmanji_broj_posade &lt;- as.data.frame(Najmanji_broj_posade, stringsAsFactors = FALSE)
        colnames(Najmanji_broj_posade) &lt;- &quot;Najmanji_broj_posade&quot;
        Prethodna_oznaka &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiPrethodnaOznakaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Prethodna_oznaka, '&quot;', &quot;]&quot;)
        Prethodna_oznaka &lt;- fromJSON(json)
        Prethodna_oznaka &lt;- as.data.frame(Prethodna_oznaka, stringsAsFactors = FALSE)
        colnames(Prethodna_oznaka) &lt;- &quot;Prethodna_oznaka&quot;
        Prethodna_luka &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiPrethodnaLukaUpisaText']/@value&quot;)
        Prethodna_luka &lt;- gsub(&quot;\&quot;&quot;, &quot;'&quot;, Prethodna_luka)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Prethodna_luka, '&quot;', &quot;]&quot;)
        Prethodna_luka &lt;- fromJSON(json)
        Prethodna_luka &lt;- as.data.frame(Prethodna_luka, stringsAsFactors = FALSE)
        colnames(Prethodna_luka) &lt;- &quot;Prethodna_luka&quot;
        Prethodna_drĹľava &lt;- xpathSApply(doc = doc, path = &quot;//input[@id = 'ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiPrethodnaDrzavaUpisaText']/@value&quot;)
        json &lt;- paste0(&quot;[&quot;, '&quot;', Prethodna_drĹľava, '&quot;', &quot;]&quot;)
        Prethodna_drĹľava &lt;- fromJSON(json)
        Prethodna_drĹľava &lt;- as.data.frame(Prethodna_drĹľava, stringsAsFactors = FALSE)
        colnames(Prethodna_drĹľava) &lt;- &quot;Prethodna_drĹľava&quot;
        
        df[[x]] &lt;- cbind(Ime, Oznaka, NIB, Vlasnik, LK_LI, br1, br2, Namjena, Vrsta_plovila, 
                         Model_plovila, Duljina_trupa, Sirina_trupa, Visina_trupa, Gaz, Nosivost, GT,
                         Snaga_motora, Brodogradiliste, Godina_gradnje, Materijal, Najveci_broj_osoba,
                         Najveci_broj_putnika, Najmanji_broj_posade, Prethodna_oznaka,
                         Prethodna_luka, Prethodna_drĹľava)
        df[[x]] &lt;- as.data.frame(df[[x]], stringsAsFactors = FALSE)
        
        df_2 &lt;- readHTMLTable(doc)
        Sys.sleep(2L)
        
        Porivni_uredjaji[[x]] &lt;- tryCatch(as.data.frame(cbind(df_2[[2]], NIB), stringsAsFactors = FALSE), error=function(e) print(paste0(&quot;Error &quot;, NIB)))
        Clanovi_posade[[x]] &lt;- tryCatch(as.data.frame(cbind(df_2[[3]], NIB), stringsAsFactors = FALSE), error=function(e) print(paste0(&quot;Error &quot;, NIB)))
        Vlasnici[[x]] &lt;- tryCatch(as.data.frame(cbind(df_2[[4]], NIB), stringsAsFactors = FALSE), error=function(e) print(paste0(&quot;Error &quot;, NIB)))
        Korisnici[[x]] &lt;- tryCatch(as.data.frame(cbind(df_2[[5]], NIB), stringsAsFactors = FALSE), error=function(e) print(paste0(&quot;Error &quot;, NIB)))
        Tereti[[x]] &lt;- cbind(remDr$findElement(using = &quot;xpath&quot;, &quot;//*/span[@id='ctl00_Content_FormContent_uiDetaljiPlovilaControl_uiTeretiLabel']&quot;)$getElementText(), NIB)
      }}
  }
  )
  
  # manipulate data after scraping
  for (i in 1:length(df)){
    if (length(df[[i]]) &lt; 13){
      df[[i]] &lt;- matrix(data = rep(NA, 26), nrow = 1, ncol = 26)
      df[[i]] &lt;- as.data.frame(df[[i]])
      colnames(df[[i]]) &lt;- c(&quot;Ime&quot;, &quot;Oznaka&quot;, &quot;NIB&quot;, &quot;Vlasnik&quot;, &quot;LK_LI&quot;, &quot;br1&quot;, &quot;br2&quot;,&quot;Namjena&quot;,
                             &quot;Vrsta_plovila&quot;, &quot;Model_plovila&quot;, &quot;Duljina_trupa&quot;, &quot;Sirina_trupa&quot;, &quot;Visina_trupa&quot;,
                             &quot;Gaz&quot;, &quot;Nosivost&quot;, &quot;GT&quot;, &quot;Snaga_motora&quot;, &quot;Brodogradiliste&quot;, &quot;Godina_gradnje&quot;, 
                             &quot;Materijal&quot;, &quot;Najveci_broj_osoba&quot;, &quot;Najveci_broj_putnika&quot;, &quot;Najmanji_broj_posade&quot;, 
                             &quot;Prethodna_oznaka&quot;, &quot;Prethodna_luka&quot;, &quot;Prethodna_drĹľava&quot;)
    }
  }
  
  df_final &lt;- do.call(rbind, df)
  df_final_1 &lt;- df_final[!is.na(df_final$NIB), ]
  </code></pre>
  <p>EDIT 2 :
  I have a problem with above code you posted. If I run:</p>
  <pre><code>(cl &lt;- (detectCores() - 1) %&gt;%  makeCluster) %&gt;% registerDoParallel
  # open a remoteDriver for each node on the cluster
  # docker run -d -p 4445:4444 selenium/standalone-chrome:3.5.3
  clusterEvalQ(cl, {
    library(RSelenium)
    remDr &lt;- remoteDriver(remoteServerAddr = &quot;192.168.99.100&quot;, port = 4445L, browserName = &quot;chrome&quot;)
    remDr$open()
  })
  myTitles &lt;- c()
  ws &lt;- foreach(x = 1:length(urls), 
                .packages = c(&quot;rvest&quot;, &quot;magrittr&quot;, &quot;RSelenium&quot;, &quot;jsonlite&quot;, &quot;htmltab&quot;, &quot;XML&quot;, &quot;RCurl&quot;))  %dopar%  {
    remDr$navigate(urls[x])
    Sys.sleep(3L)
    remDr$getTitle()[[1]]
                }
  </code></pre>
  <p>it returns an error</p>
  <pre><code>Error in { : task 1 failed - &quot;   Summary: UnknownError
       Detail: An unknown server-side error occurred while processing the command.
       Further Details: run errorDetails method&quot;
  </code></pre>