title: Create a file to keep track of task progress
body: <p>Running a script for datasets transfer that requires nearly 3-4months to complete via ssh. Unfortunately, connection break after 6-8days so a restart required.</p>
  
  <p>Script:</p>
  
  <pre><code>import psycopg2
  from time import sleep
  from config import config
  from tqdm import tqdm
  import requests
  import json
  import subprocess
  
  subprocess.call("./airquality.sh", shell=True)
  
  def val_json():
      db = "select to_json(d) from (  select \
          a.particles_data as particles, \
          a.o3_data as \"O3\", \
          to_timestamp(a.seconds) as \"dateObserved\", \
          l.description as name, \
              json_build_object( \
                  'coordinates', \
                  json_build_array(l.node_lon, l.node_lat) \
              ) as location \
          from airquality as a \
              inner join deployment as d on \
                  d.deployment_id = a.deployment_id \
              inner join location as l on \
                  l.location_id = d.location_id \
      ) as d"
      return db
  
  def main():
  
      url = 'http://localhost:1026/v2/entities/003/attrs?options=keyValues'
      headers = {"Content-Type": "application/json", \
                 "fiware-service": "urbansense",  \
                 "fiware-servicepath": "/basic"}
      conn = None
      try:
          params = config()
          with psycopg2.connect(**params) as conn:
              with conn.cursor(name='my_cursor') as cur:
                  cur.itersize = 2000
                  cur.execute(val_json())
         # row = cur.fetchone()
                  for row in tqdm(cur):
                      jsonData = json.dumps(row)
                      if jsonData.startswith('[') and jsonData.endswith(']'):
                          jsonData = jsonData[1:-1]
                          print(jsonData)
                      requests.post(url, data= jsonData, headers=headers)
                      sleep(1)
  
                  cur.close()
      except (Exception, psycopg2.DatabaseError) as error:
          print(error)
      finally:
          if conn is not None:
              conn.close()
  
  if __name__ == '__main__':
      main()
  </code></pre>
  
  <p>How do I create a file and keep track of transfer progress, so when this script is run again (after a connection lost), datasets are fetched from where it stops previously?</p>
  
  <p><strong>EDIT:</strong></p>
  
  <p>Oops! I'm lost somewhere along the way.
  I managed to have the scripts run and write progress to a text file (<code>air.txt</code>) which I manually created with content <code>0</code> (otherwise script won't run at all).
  When this script run, content of <code>air.txt</code> file gets updated with cursor position value.</p>
  
  <p><strong>Problem:</strong></p>
  
  <p>I problem now is that when I stopped the script running (as a way of checking), and restarts again to make sure it picks from previous position, scripts start from <code>0</code> again overwriting previous value (and start a new count, instead of reading that as start position). 
  Below is my updated script:</p>
  
  <pre><code>def val_json():
      db = "select to_json(d) from (  select \
          a.particles_data as particles, \
          a.o3_data as \"O3\", \
          to_timestamp(a.seconds) as \"dateObserved\", \
          l.description as name, \
              json_build_object( \
                  'coordinates', \
                  json_build_array(l.node_lon, l.node_lat) \
              ) as location \
          from airquality as a \
              inner join deployment as d on \
                  d.deployment_id = a.deployment_id \
              inner join location as l on \
                  l.location_id = d.location_id \
      ) as d"
      return db
  
  def main():
      RESTART_POINT_FILE = 'air.txt'
      conn = None
      try:
          params = config()
          with open(RESTART_POINT_FILE) as fd:
             rows_to_skip = int(next(fd))
      #except OSError:
          rows_to_skip = 0
          with psycopg2.connect(**params) as conn:
              with conn.cursor(name='my_cursor') as cur:
                  cur.itersize = 2000
                  cur.execute(val_json())
  
                  for processed_rows, row in enumerate(tqdm(cur)):
                      if processed_rows &lt; rows_to_skip: continue
                      jsonData = json.dumps(row)
                      if jsonData.startswith('[') and jsonData.endswith(']'):
                          jsonData = jsonData[1:-1]
  
                          print('\n', processed_rows, '\t', jsonData)
                      #update progress file...
                      with open(RESTART_POINT_FILE, "w") as fd:
                          print(processed_rows, file=fd)
                      sleep(1)
  
                  cur.close()
      except (Exception, psycopg2.DatabaseError) as error:
          print(error)
  
      finally:
          if conn is not None:
              conn.close()
  
  if __name__ == '__main__':
      main()
  </code></pre>