title: R: re-writing a loop using the "doparallel", "foreach" and "purrr" libraries
body: <p>I wrote the following code in R which performs (loop) a series of data manipulation operations on some artificially generated data (the final output is called &quot;final_results&quot;):</p>
  <pre><code>#load library
      library(dplyr)
  
  library(data.table)
  
  set.seed(123)
  
  # create some data for this example
  a1 = rnorm(1000,100,10)
  b1 = rnorm(1000,100,5)
  c1 = sample.int(1000, 1000, replace = TRUE)
  train_data = data.frame(a1,b1,c1)
  
  
  ####
  results_table &lt;- data.frame()
  
  for (i in 1:10 ) {
      
      #generate random numbers
      random_1 =  runif(1, 80, 120)
      random_2 =  runif(1, random_1, 120)
      random_3 =  runif(1, 85, 120)
      random_4 =  runif(1, random_3, 120)
      
      #bin data according to random criteria
      train_data &lt;- train_data %&gt;% mutate(cat = ifelse(a1 &lt;= random_1 &amp; b1 &lt;= random_3, &quot;a&quot;, ifelse(a1 &lt;= random_2 &amp; b1 &lt;= random_4, &quot;b&quot;, &quot;c&quot;)))
      
      train_data$cat = as.factor(train_data$cat)
      
      #new splits
      a_table = train_data %&gt;%
          filter(cat == &quot;a&quot;) %&gt;%
          select(a1, b1, c1, cat)
      
      b_table = train_data %&gt;%
          filter(cat == &quot;b&quot;) %&gt;%
          select(a1, b1, c1, cat)
      
      c_table = train_data %&gt;%
          filter(cat == &quot;c&quot;) %&gt;%
          select(a1, b1, c1, cat)
      
      split_1 =  runif(1,0, 1)
      split_2 =  runif(1, 0, 1)
      split_3 =  runif(1, 0, 1)
      
      #calculate 60th quantile (&quot;quant&quot;) for each bin
      
      table_a = data.frame(a_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_1)))
      
      table_b = data.frame(b_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_2)))
      
      table_c = data.frame(c_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_3)))
      
      
      
      
      #create a new variable (&quot;diff&quot;) that measures if the quantile is bigger tha the value of &quot;c1&quot;
      table_a$diff = ifelse(table_a$quant &gt; table_a$c1,1,0)
      table_b$diff = ifelse(table_b$quant &gt; table_b$c1,1,0)
      table_c$diff = ifelse(table_c$quant &gt; table_c$c1,1,0)
      
      #group all tables
      
      final_table = rbind(table_a, table_b, table_c)
      
      #create a table: for each bin, calculate the average of &quot;diff&quot;
      final_table_2 = data.frame(final_table %&gt;%
                                     group_by(cat) %&gt;%
                                     summarize(
                                         mean = mean(diff)
                                     ))
      
      #add &quot;total mean&quot; to this table
      final_table_2 = data.frame(final_table_2 %&gt;% add_row(cat = &quot;total&quot;, mean = mean(final_table$diff)))
      
      #format this table: add the random criteria to this table for reference
      final_table_2$random_1 = random_1
      
      final_table_2$random_2 = random_2
      
      final_table_2$random_3 = random_3
      
      final_table_2$random_4 = random_4
      
      final_table_2$split_1 = split_1
      
      final_table_2$split_2 = split_2
      
      final_table_2$split_3 = split_3
      
      final_table_2$iteration_number = i
      
      
      results_table &lt;- rbind(results_table, final_table_2)
      
      final_results = dcast(setDT(results_table), iteration_number + random_1 + random_2 + random_3 + random_4 + split_1 + split_2 + split_3 ~ cat, value.var = 'mean')
      
  } 
  </code></pre>
  <p>The above loop works perfectly fine - but I am trying to learn more about R and trying to re-write this loop using other functions from other libraries such as the <code>&quot;doParallel&quot;, &quot;foreach&quot;</code> and <code>&quot;purrr&quot;</code> libraries.</p>
  <p><strong>Option 1:</strong></p>
  <p>I came across the following code in R which shows the general template for writing loops using the &quot;purrr&quot; library (apparently &quot;map_df&quot; is a function that uses the code from the loop):</p>
  <pre><code>#option 1
  library(dplyr)
  library(purrr)
  library(tictoc)
  
  
  data_gen &lt;- function(){ #here you insert your data generating process
    tibble(
      x = runif(100),
      y = runif(100)
    )
  }
  
  N &lt;- 10000 #number of datasets do be generated
  
  
  tic('method A')  #not necessary, measures the time of the code between 'tic' and 'toc'
  output &lt;- tibble(
    i = 1:N
  ) %&gt;%
    split(.$i) %&gt;%
    map_df(
      ~data_gen()
    )
  toc()
  </code></pre>
  <p>However, I am not sure how this code can be adapted to fit my example. I first created the <code>map_df</code> function:</p>
  <pre><code>#create map_df function:
  
  map_df &lt;- function() {
      #bin data according to random criteria
      train_data &lt;- train_data %&gt;% mutate(cat = ifelse(a1 &lt;= random_1 &amp; b1 &lt;= random_3, &quot;a&quot;, ifelse(a1 &lt;= random_2 &amp; b1 &lt;= random_4, &quot;b&quot;, &quot;c&quot;)))
  
      train_data$cat = as.factor(train_data$cat)
  
      #new splits
      a_table = train_data %&gt;%
          filter(cat == &quot;a&quot;) %&gt;%
          select(a1, b1, c1, cat)
  
      b_table = train_data %&gt;%
          filter(cat == &quot;b&quot;) %&gt;%
          select(a1, b1, c1, cat)
  
      c_table = train_data %&gt;%
          filter(cat == &quot;c&quot;) %&gt;%
          select(a1, b1, c1, cat)
  
      split_1 =  runif(1,0, 1)
      split_2 =  runif(1, 0, 1)
      split_3 =  runif(1, 0, 1)
  
      #calculate 60th quantile (&quot;quant&quot;) for each bin
  
      table_a = data.frame(a_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_1)))
  
      table_b = data.frame(b_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_2)))
  
      table_c = data.frame(c_table%&gt;% group_by(cat) %&gt;%
                               mutate(quant = quantile(c1, prob = split_3)))
  
  
  
  
      #create a new variable (&quot;diff&quot;) that measures if the quantile is bigger tha the value of &quot;c1&quot;
      table_a$diff = ifelse(table_a$quant &gt; table_a$c1,1,0)
      table_b$diff = ifelse(table_b$quant &gt; table_b$c1,1,0)
      table_c$diff = ifelse(table_c$quant &gt; table_c$c1,1,0)
  
      #group all tables
  
      final_table = rbind(table_a, table_b, table_c)
  
      #create a table: for each bin, calculate the average of &quot;diff&quot;
      final_table_2 = data.frame(final_table %&gt;%
                                     group_by(cat) %&gt;%
                                     summarize(
                                         mean = mean(diff)
                                     ))
  
      #add &quot;total mean&quot; to this table
      final_table_2 = data.frame(final_table_2 %&gt;% add_row(cat = &quot;total&quot;, mean = mean(final_table$diff)))
  
      #format this table: add the random criteria to this table for reference
      final_table_2$random_1 = random_1
  
      final_table_2$random_2 = random_2
  
      final_table_2$random_3 = random_3
  
      final_table_2$random_4 = random_4
  
      final_table_2$split_1 = split_1
  
      final_table_2$split_2 = split_2
  
      final_table_2$split_3 = split_3
  
      final_table_2$iteration_number = i
  
  
      results_table &lt;- rbind(results_table, final_table_2)
  
      final_results = dcast(setDT(results_table), iteration_number + random_1 + random_2 + random_3 + random_4 + split_1 + split_2 + split_3 ~ cat, value.var = 'mean')
  }
  </code></pre>
  <p>But when I try to run the general template, it produces the following error:</p>
  <pre><code>data_gen &lt;- function(){ #here you insert your data generating process
      tibble(
          # create some data for this example
          a1 = rnorm(1000,100,10)
          b1 = rnorm(1000,100,5)
          c1 = sample.int(1000, 1000, replace = TRUE)
          train_data = data.frame(a1,b1,c1)
      )
  }
  
  N &lt;- 10000 #number of datasets do be generated
  
  
  tic('method A')  #not necessary, measures the time of the code between 'tic' and 'toc'
  output &lt;- tibble(
      i = 1:N
  ) %&gt;%
      split(.$i) %&gt;%
      map_df(
          ~data_gen()
      )
  toc() 
  
  Error in map_df(., ~data_gen()) : unused arguments (., ~data_gen())
  </code></pre>
  <p>Does anyone know why this error is being produced?</p>
  <p><strong>Option 2</strong> :</p>
  <p>I am not sure how the &quot;doParallel&quot; and &quot;foreach&quot; libraries can be used in my example. It seems that all examples with &quot;doParallel&quot; require the user to begin by defining the number of &quot;cores&quot; they wish the computer will use:</p>
  <pre><code> library(doParallel)
   cl &lt;- makeCluster(2)
   registerDoParallel(cl)
  </code></pre>
  <p>And in the end, the user has to instruct the computer to stop the process:</p>
  <pre><code>stopCluster(cl)
  </code></pre>
  <p>Beyond this, I am not sure how the &quot;doParalell&quot; and the &quot;foreach&quot; library can be used to benefit my example.</p>
  <p>Can someone please show me this?
  Thanks</p>