title: Batch insertion into postgreSQL streaming from local file using JavaScript
body: <p>I need to insert roughly 8 million rows into a postgreSQL database from a local file with JSON objects. The code below which inserts one row at a time works, but it's awfully slow.</p>
  
  <p>The JSON file is too big to store in memory (even with <code>--max-old-space-size=8192</code>), which is I why turned to streaming. However, I'm not sure the best way to read and insert rows in batches of e.g. 10000. Can anyone provide suggestions?</p>
  
  <pre><code>const Pool = require("pg").Pool
  const fs = require("fs")
  const StreamArray = require("stream-json/utils/StreamArray")
  let stream = StreamArray.make()
  let jsonPath = "dataset.json"
  
  let completionCounter = 0
  
  // login details removed
  const pool = new Pool({
    user: myUser,
    host: myHost,
    port: 5432,
    database: myDB,
    password: myPwd,
    ssl: true
  })
  
  // Event listener for each array in the stream
  stream.output.on("data", (arrayItem) =&gt; {
    let data = arrayItem.value
    pool
      .query(
        `INSERT INTO link_list (id, url, status) VALUES($1, $2, $3)`,
        [data.id, data.url, data.status])
      .then(() =&gt; {
        completionCounter++
        if (completionCounter % 1000 == 0) {
          console.log("entries completed: ", completionCounter)
        }
      }).catch((err) =&gt; {
        console.log(err)
      })
  })
  
  stream.output.on("end", () =&gt; {
    console.log("JSON file stream has ended");
  })
  
  // Start the stream
  console.log("Starting to read data...")
  fs.createReadStream(jsonPath).pipe(stream.input)
  </code></pre>