title: how to put data in mongodb
body: <p>I'm calling for your help today because I can't put the data in mongo db, indeed, I managed to connect, but impossible to add the data.</p>
  <p>I'm going to explain you a little bit my code, in a first time, I'm going to search the pages according to a start number and an end number, the first function is going to search if the page exists or not, in my 2nd function search, then I'm going to search the category I'm interested in, then I'm going to get the information of this page, that is to say what you can see in the &quot;const contact&quot;, then I'm going to add all this in a db file.</p>
  <p>So what I want is instead of putting it in the db file it automatically adds me to mongodb.</p>
  <pre><code>// import
  const Datastore = require('nedb')
  // const fs = require('fs')
  const puppeteer = require('puppeteer');
  const fs = require('fs');
  const { resolve } = require('path');
  
  // global variables
  var totalFound = 0
  var headless = true
  const DB_PATH = './huarenjietest2.db' //addition in the db file
  const START_LINK = 'https://www.huarenjiewang.com/info-[pageIndex].html'
  const LOCATION_SeleCTOR = 'body &gt; div.body1000 &gt; div.location'
  const COOKIE_SELECTOR = 'body &gt; div.fc-consent-root &gt; div.fc-dialog-container &gt; div.fc-dialog.fc-choice-dialog &gt; div.fc-dialog-content &gt; div &gt; div.fc-header.fc-dialog-restricted-content &gt; h1'
  const NotFoundSelector = 'body &gt; div &gt; div.tip-bgC &gt; div'
  const phoneSelector = '#show-phone'
  const workTypeSelector = 'body &gt; div.body1000 &gt; div.wrapper &gt; div.information_bd &gt; div.bd_left &gt; div.extra_contact &gt; div &gt; ul &gt; li:nth-child(4)'
  const zoneSelector =  'body &gt; div.body1000 &gt; div.wrapper &gt; div.information_bd &gt; div.bd_left &gt; div.extra_contact &gt; div &gt; ul &gt; li:nth-child(1)'
  const whoSelector = 'body &gt; div.body1000 &gt; div.wrapper &gt; div.information_bd &gt; div.bd_left &gt; div.extra_contact &gt; div &gt; ul &gt; li:nth-child(6) &gt; span.faburen &gt; a:nth-child(1)'
  const infoSelector = 'body &gt; div.body1000 &gt; div.wrapper &gt; div.information_bd &gt; div.bd_left &gt; div.view_bd &gt; p'
  const dateSeloctor = 'body &gt; div.body1000 &gt; div.wrapper &gt; div.information_bd &gt; div.bd_left &gt; div.information_hd &gt; div.my_yp_view_title2 &gt; div.my_yp_view_title2_l'
  const COOKIE_BUTTON = 'body &gt; div.fc-consent-root &gt; div.fc-dialog-container &gt; div.fc-dialog.fc-choice-dialog &gt; div.fc-footer-buttons-container &gt; div.fc-footer-buttons &gt; button.fc-button.fc-cta-consent.fc-primary-button'
  
  //mongodb
  var express = require(&quot;express&quot;)
  const database = require('./config/database')
  var router = express.Router();
  const Huarenjiemongo = require(&quot;./model/huarenjiemodel&quot;)
  
  database.connection()
  
  // command line options
  const optionDefinitions = [
    { name: 'headless', type: Boolean },
    { name: 'start', type: Number },
    { name: 'end', type: Number },
    { name: 'export', type: String },
  ]
  const commandLineArgs = require('command-line-args');
  const { exit } = require('process');
  const { mongo } = require('mongoose');
  const options = commandLineArgs(optionDefinitions)
  
  const startIndex = options.start
  const endIndex = options.end
  const outputPath = options.export
  if(!startIndex || !endIndex){
    console.log('use command like : node huarenjie.js --start 12321323 --end 13321323 --export output.csv')
    exit()
  }
  
  const dbPath = DB_PATH
  const db = new Datastore({ filename: dbPath, autoload: true })
  db.ensureIndex({ fieldName: 'infoIndex', unique: true }, function (err) {
    if (err) {
      console.log('ensureIndex error :', err)
    }
  })
  
  
   function dateyyyyMMdd(d, separate = '') {
    let date = d
    
  
    var mm = date.getMonth() + 1 // getMonth() is zero-based
    var dd = date.getDate()
  
    return [date.getFullYear(), (mm &gt; 9 ? '' : '0') + mm, (dd &gt; 9 ? '' : '0') + dd].join(separate)
  }
  
  database.connection();
  
  function delay(time) {
     return new Promise(function(resolve) { 
         setTimeout(resolve, time)
     });
  }
  
  async function checkPageExite(page){
    return new Promise((resolve, reject) =&gt; {
      page.waitForSelector(LOCATION_SeleCTOR).then(() =&gt; {
        resolve(true)
      }).catch(()=&gt; { resolve(false)})
  
      page.waitForSelector(NotFoundSelector).then(() =&gt; {
        console.info('page not exist')
        resolve(false)
      }).catch(()=&gt; { resolve(false)})
  
      page.waitForSelector(COOKIE_BUTTON).then(() =&gt; {
        return Promise.resolve()
      }).then(() =&gt; {
        return page.$eval( COOKIE_BUTTON, form =&gt; form.click() )
      }).then(() =&gt; {
        if(page.url() === 'https://www.huarenjiewang.com/changecity.php'){
          resolve(false)
        } else {
          resolve(true)
        }
      }).catch(()=&gt; { resolve(false)})
    })
  }
  
  async function search() {
    const browser = await puppeteer.launch({ headless: headless })
    console.log('====&gt; Puppeteer started')
    const page = await browser.newPage();
    await page.setViewport({ width: 1280, height: 800 })
    for(var i=startIndex;i&lt;=endIndex;i++){
      try {
        console.info(`====== index : ${i}  ========`)
        await page.goto(START_LINK.replace('[pageIndex]',i))
        // await page.waitForSelector(LOCATION_SeleCTOR)
        const exist = await checkPageExite(page)
        if(exist === 'skip'){
          continue
        }
        
        if(exist === true){
          let element = await page.$(LOCATION_SeleCTOR)
          let categoryValue = await page.evaluate(el =&gt; el.textContent, element)
          if(categoryValue.includes('appartement/louer')){
            let phone = await page.$(phoneSelector)
            let phoneValue = await page.evaluate(el =&gt; el.getAttribute(&quot;data-phone&quot;), phone)
            if(phoneValue){
              let workElement = await page.$(workTypeSelector)
              let workValue = await page.evaluate(el =&gt; el.textContent, workElement)
              let zoneElement = await page.$(zoneSelector)
              let zoneValue = await page.evaluate(el =&gt; el.textContent, zoneElement)
              let whoElement = await page.$(whoSelector)
              let whoValue = await page.evaluate(el =&gt; el.textContent, whoElement)
              let infoElement = await page.$(infoSelector)
              let infoValue = await page.evaluate(el =&gt; el.textContent, infoElement)
              let dateElement = await page.$(dateSeloctor)
              var dateValue = await page.evaluate(el =&gt; el.textContent, dateElement)
              var date 
              var dateStr
              var reg1 = /(\d{4}-\d{2}-\d{2})/
              const found1 = dateValue.match(reg1)
  
             
  
              if(found1 &amp;&amp; found1.length &gt; 1){
                dateStr = found1[1]
              }
  
              if(dateValue.includes('昨天')){
                var date = new Date()
                date.setDate(date.getDate() -1)
                dateStr = dateyyyyMMdd(date,'-')
              }
  
              if(dateValue.includes('前天')){
                var date = new Date()
                date.setDate(date.getDate() -2)
                dateStr = dateyyyyMMdd(date,'-')
              }
              
              const contact = {
                phone : phoneValue,
                zone: zoneValue,
                workType: workValue,
                who: whoValue,
                info: infoValue,
                date: dateStr || '',
                infoIndex: i
              }
              db.update({ infoIndex: i }, { $set: contact }, { upsert: true }, function () {});
              console.info(`${i}#############&gt;saved`)
            } else {
              console.info(`${i}=&gt;skip==&gt;phone number missing`)
            }
                    } else {
            console.info(`${i}=&gt;skip==&gt;category not correct`)
          }
          if(categoryValue.includes('maison/louer')){
            let phone = await page.$(phoneSelector)
            let phoneValue = await page.evaluate(el =&gt; el.getAttribute(&quot;data-phone&quot;), phone)
            if(phoneValue){
              let workElement = await page.$(workTypeSelector)
              let workValue = await page.evaluate(el =&gt; el.textContent, workElement)
              let zoneElement = await page.$(zoneSelector)
              let zoneValue = await page.evaluate(el =&gt; el.textContent, zoneElement)
              let whoElement = await page.$(whoSelector)
              let whoValue = await page.evaluate(el =&gt; el.textContent, whoElement)
              let infoElement = await page.$(infoSelector)
              let infoValue = await page.evaluate(el =&gt; el.textContent, infoElement)
              let dateElement = await page.$(dateSeloctor)
              var dateValue = await page.evaluate(el =&gt; el.textContent, dateElement)
              var date 
              var dateStr
              var reg1 = /(\d{4}-\d{2}-\d{2})/
              const found1 = dateValue.match(reg1)
  
              if(found1 &amp;&amp; found1.length &gt; 1){
                dateStr = found1[1]
              }
  
              if(dateValue.includes('昨天')){
                var date = new Date()
                date.setDate(date.getDate() -1)
                dateStr = dateyyyyMMdd(date,'-')
              }
  
              if(dateValue.includes('前天')){
                var date = new Date()
                date.setDate(date.getDate() -2)
                dateStr = dateyyyyMMdd(date,'-')
              }
              
              const contact = {
                phone : phoneValue,
                zone: zoneValue,
                workType: workValue,
                who: whoValue,
                info: infoValue,
                date: dateStr || '',
                infoIndex: i
              }
             
              db.update({ infoIndex: i }, { $set: contact }, { upsert: true }, function () {})
              console.info(`${i}#############&gt;saved-tech`)
            } else {
              console.info(`${i}=&gt;skip==&gt;phone number missing`)
            }
                    } else {
            console.info(`${i}=&gt;skip==&gt;category not correct`)
          }
        }
        
        // await delay(5000)
      } catch (error) {
        console.info(`${i}=&gt;error read page:`, error)
      }
        
    }
  
  }
  
  const connectToMongoDB = async() =&gt; {
    await database().then(async (mongoose) =&gt; {
      try {
        console.log('Connecto to mongodb')
        db.collection({ infoIndex: i }, { $set: contact }, { upsert: true }, function () {})
        await new huarenjieShema(contact).save()
      } finally {
        mongoose.connection.close()
      }
    })
  }
  connectToMongoDB()
  
  async function main() {
    if(outputPath){
      db.find({ &quot;infoIndex&quot;: { $gte: startIndex , $lte: endIndex} }, function (err, docs) {
      if(err){
        console.error(err) 
      } else {
        var body = `infoIndex;who;phone;zone;workType;comment\n`
        for(var doc of docs){
          body += `${doc.infoIndex};${(doc.who || '').trim()};${doc.phone};${(doc.zone || '').trim()};${(doc.workType || '').trim()};${(doc.info || '').trim()}\n`
        }
        fs.writeFileSync(options.export, body)
      }
      exit()
    });
  } else {
    await search()
  
  }
    console.info('&gt;&gt;&gt;&gt;&gt; Finished')
    console.info('&gt;&gt;&gt;&gt;&gt; totalFound:', totalFound)
  }
  
  
  
  
  main()
  </code></pre>
  <p>here is the model</p>
  <pre><code>const mongoose = require('mongoose')
  
  const  huarenjieShenma = mongoose.Schema({
  
      infoIndex: {
          type: Number,
          unique: true,
      },
  
      who: {
          type: String,
      },
  
      phone: {
          type: Number,
      },
  
      zone: {
          type: String,
      },
  
      workType: {
          type: String,
      },
  
      comment: {
          type: String,
      },
  
  })
  
  module.exports = mongoose.model('Huarenjie', huarenjieShenma)
  </code></pre>