title: Iterative pairwise comparisons across columns in R
body: <p>I have a very large matrix 200 x 1500, where the rows are samples and the columns are data. I want to do pairwise comparisons of all 1500 columns (~1.1M tests), so <code>combn</code> would take too long. I'm trying to run a linear model on the first column against each 1499 other columns, process/write the pvalue to output file (i.e.data1.tsv) and then repeat for the second column (excluding the comparison with the first column) and save to data2.tsv. And continue until all comparisons have been made. Below is the code for what I am trying to achieve.</p>
  <pre><code>library(data.table)
  
  df = as.data.frame(matrix(runif(20, min=0, max=100), nrow=4))
  colnames(df) = c(&quot;data_1&quot;, &quot;data_2&quot;, &quot;data_3&quot;, &quot;data_4&quot;, &quot;data_5&quot;)
  rownames(df) = c(&quot;sample_1&quot;, &quot;sample_2&quot;, &quot;sample_3&quot;, &quot;sample_4&quot;)
  
  pval_1 = as.numeric(summary(lm(data_1 ~ data_2, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_2 = as.numeric(summary(lm(data_1 ~ data_3, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_3 = as.numeric(summary(lm(data_1 ~ data_4, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_4 = as.numeric(summary(lm(data_1 ~ data_5, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  
  data_1 = data.frame(id1 = c(&quot;data_1&quot;,&quot;data_1&quot;,&quot;data_1&quot;,&quot;data_1&quot;),
                       id2 = c(&quot;data_2&quot;,&quot;data_3&quot;,&quot;data_4&quot;,&quot;data_5&quot;),
                        pval = c(pval_1, pval_2,pval_3,pval_4))
  
  fwrite(data_1, to_path)
  
  pval_5 = as.numeric(summary(lm(data_2 ~ data_3, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_6 = as.numeric(summary(lm(data_2 ~ data_4, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_7 = as.numeric(summary(lm(data_2 ~ data_5, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  
  data_2 = data.frame(id1 = c(&quot;data_2&quot;,&quot;data_2&quot;,&quot;data_2&quot;),
                       id2 = c(&quot;data_3&quot;,&quot;data_4&quot;,&quot;data_5&quot;),
                        pval = c(pval_5, pval_6,pval_7))
  
  fwrite(data_2, to_path)
  
  pval_8 = as.numeric(summary(lm(data_3 ~ data_4, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  pval_9 = as.numeric(summary(lm(data_3 ~ data_5, data=df))$coefficients[,&quot;Pr(&gt;|t|)&quot;][2])
  
  data_3 = data.frame(id1 = c(&quot;data_3&quot;,&quot;data_3&quot;),
                       id2 = c(&quot;data_4&quot;,&quot;data_5&quot;),
                        pval = c(pval_8,pval_9))
  
  fwrite(data_3, to_path )
  </code></pre>