title: Instagram API - paging through 'new' posts
body: <p>So I'm using node.js and the module instagram-node-lib to download metadata for Instagram posts. I have a couple of hashtags that I want to search for, and I want to download all existing posts (handling request failure during pagination) as well as monitor all new posts.</p>
  
  <p>I have managed to crack the first part - downloading all existing posts and handling failure (I noticed that sometimes the Instagram API would just fail on me, so I've added redundancy to remember the last successful page I downloaded and attempt again from that point). For anyone who is interested, here is my code (note, I use Postgres to save the posts, and I've abbreviated/obfuscated some of the code for ease of reading and for commercial purposes) **apologies for the length of code, but I think this will come in useful to someone:</p>
  
  <pre><code>var db = new (require('./postgres'))
      ,api = require("instagram-node-lib")
  ;
  
  var HASHTAGS = ["fluffy", "kittens"] //this is just an example!
      ,CLIENT_ID = "YOUR_CLIENT_ID"
      ,CLIENT_SECRET = "YOUR_CLIENT_SECRET"
      ,HOST = "https://api.instagram.com"
      ,PORT = 443
      ,PATH = "/v1/media/popular?client_id=" + CLIENT_ID
  ;
  
  var hashtagIndex = 0
      ,settings
  ;
  
  /**
   * Initialise the module for use
   */
  exports.initialise = function(){
      api.set("client_id", CLIENT_ID);
      api.set("client_secret", CLIENT_SECRET);
  
      if( !settings){
          settings = {
              hashtags: [] 
          }
  
          for( var i in HASHTAGS){
              settings.hashtags[i] = {
                  name: HASHTAGS[i],
                  maxTagId: null,
                  minTagId: null,
                  nextMaxTagId: null,
              }
          }
      }
      // console.log(settings);
  
      db.initialiseSettings(); //I haven't included the code for this - basically just loads settings from the database, overwriting the defaults above if they exist, otherwise it creates them using the above object. I store the settings as a JSON object in the DB and parse them on load
  
      execute();    
  }
  
  function execute(){
      var params = {
          name: HASHTAGS[hashtagIndex],
          complete: function(data, pagination){
              var hashtag = settings.hashtags[hashtagIndex];
  
              //from scratch
              if( !hashtag.maxTagId){
                  console.log('Downloading old posts from scratch');
  
                  getOldPosts();
              }
              //still loading old (previously failed)
              else if( hashtag.nextMaxTagId){
                  console.log('Downloading old posts from last saved position');
  
                  getOldPosts(hashtag.nextMaxTagId);
              }
              //new posts only
              else {
                  console.log('Downloading new posts only');
  
                  getNewPosts(hashtag.minTagId);
              }
          },
          error: function(msg, obj, caller){
              apiError(msg, obj, caller);
          }
      }
  
      api.tags.info(params);    
  }
  
  function getOldPosts(maxTagId){
      console.log();
  
      var params = {
          name: HASHTAGS[hashtagIndex],
          count: 100,
          max_tag_id: maxTagId || undefined,
          complete: function(data, pagination){
              console.log(pagination); 
  
              var hashtag = settings.hashtags[hashtagIndex];
  
              //reached the end
              if( pagination.next_max_tag_id == hashtag.maxTagId){
                  console.log('Downloaded all posts for #' + HASHTAGS[hashtagIndex]);
  
                  hashtag.nextMaxTagId = null; //reset nextMaxTagId - that way next time we execute the script we know to just look for new posts
                  saveSettings(function(){
                      next();
                  }); //Another function I haven't include - just saves the settings object, overwriting what is in the database. Once saved, executes the next() function
              }
              else {
                  //from scratch
                  if( !hashtag.maxTagId){
                      //these values will be saved once all posts in this batch have been saved. We set these only once, meaning that we have a baseline to compare to - enabling us to determine if we have reached the end of pagination
                      hashtag.maxTagId = pagination.next_max_tag_id;
                      hashtag.minTagId = pagination.min_tag_id;                    
                  }
  
                  //if there is a failure then we know where to start from - this is only saved to the database once the posts are successfully saved to database
                  hashtag.nextMaxTagId = pagination.next_max_tag_id;   
  
                  //again, another function not included. saves the posts to database, then updates the settings. Once they have completed we get the next page of data
                  db.savePosts(data, function(){
                      saveSettings(function(){
                          getOldPosts(hashtag.nextMaxTagId);         
                      });
                  });
              }                        
          },
          error: function(msg, obj, caller){
              apiError(msg, obj, caller);
              //keep calm and try again - this is our failure redundancy
              execute();    
          }
      }
  
      var posts = api.tags.recent(params);
  }
  
  /**
   * Still to be completed!
   */
  function getNewPosts(minTagId){
  
  }
  
  function next(){
      if( hashtagIndex &lt; HASHTAGS.length - 1){
          console.log("Moving onto the next hashtag...");
  
          hashtagIndex++;
  
          execute(); 
      }   
      else {
          console.log("All hashtags processed...");
      } 
  }
  </code></pre>
  
  <p>Ok so here is my dilema about solving the next piece of the puzzle - downloading new posts (in other words, only those new posts that have come into existence since I last downloaded all the posts). Should I use <a href="http://instagram.com/developer/realtime/" rel="nofollow">Instagram subscriptions</a> or is there a way to implement paging similar to what I've already used? I'm worried that if I use the former solution then if there is a problem with my server and it goes down for a period of time then I will miss out on some posts. I' worried that if I use the latter solution then it might not be possible to page through the records, because is the Instagram API set up to enable forward paging rather than backward paging?</p>
  
  <p>I've attempted to post questions in the Google Instagram API Developers Group a couple of times and none of my messages seem to be appearing in the forum so I thought I'd resort to trusty stackoverflow</p>