AppController/djinn.rb
# All nodes will compare the list of AppServers they should be running,
# with the list of AppServers actually running, and make the necessary
# adjustments. Effectively only login node and appengine nodes will run
# AppServers (login node runs the dashboard).
All nodes will compare the list of AppServers they should be running,
with the list of AppServers actually running, and make the necessary
adjustments. Effectively only login node and appengine nodes will run
AppServers (login node runs the dashboard).
All nodes will compare the list of AppServers they should be running with the list of AppServers actually running and make the necessary adjustments . Effectively only login node and appengine nodes will run AppServers ( login node runs the dashboard ) .
def check_running_apps()
    if AMS_LOCK.locked?
      Djinn.log_debug("Another thread already working with appmanager.")
      return
    end
    CronHelper.clear_app_crontabs() unless my_node.is_shadow?
    to_start = []
    no_appservers = []
    my_apps = []
    to_end = []
    APPS_LOCK.synchronize {
      @app_info_map.each { |app, info|
        next unless info['appengine']
        if my_node.is_shadow?
          if !info['nginx'].nil? and !info['language'].nil?
            CronHelper.update_cron(get_load_balancer_ip(),
              info['nginx'], info['language'], app)
          end
        end
        Djinn.log_debug("Checking #{app} with appengine #{info}.")
        info['appengine'].each { |location|
          host, port = location.split(":")
          next if @my_private_ip != host
          if Integer(port) < 0
            to_start << app
            no_appservers << app
          elsif not MonitInterface.is_running?("#{app}-#{port}")
            Djinn.log_warn("Didn't find the AppServer for #{app} at port #{port}.")
            to_end << "#{app}:#{port}"
          else
            my_apps << "#{app}:#{port}"
          end
        }
      }
    }
    my_apps.each { |appserver|
      app, _ = appserver.split(":")
      no_appservers.delete(app)
    }
    Djinn.log_debug("Running AppServers on this node: #{my_apps}.") unless my_apps.empty?
    MonitInterface.running_appengines().each { |appengine|
      next if my_apps.include?(appengine)
      app, _ = appengine.split(":")
      time_to_delete = true
      if @last_decision[app]
        if Time.now.to_i - @last_decision[app] < APP_UPLOAD_TIMEOUT * RETRIES
          time_to_delete = false
        end
      end
      if to_start.include?(app) and !time_to_delete
        Djinn.log_debug("Ignoring request for #{app} since we have pending AppServers.")
        to_start.delete(app)
        no_appservers.delete(app)
      else
        to_end << appengine
      end
    }
    Djinn.log_debug("First AppServers to start: #{no_appservers}.") unless no_appservers.empty?
    Djinn.log_debug("AppServers to start: #{to_start}.") unless to_start.empty?
    Djinn.log_debug("AppServers to terminate: #{to_end}.") unless to_end.empty?
    Thread.new {
      AMS_LOCK.synchronize {
        Djinn.log_debug("Checking if any app need to be restarted.")
        apps_to_restart = []
        app_language = ""
        APPS_LOCK.synchronize {
          apps_to_restart = @apps_to_restart
        }
        apps_to_restart.each { |app_name|
          Djinn.log_info("Got #{app_name} to restart (if applicable).")
          APPS_LOCK.synchronize {
            next unless @apps_to_restart.include?(app_name)
            setup_appengine_application(app_name)
            app_language = @app_info_map[app_name]['language']
          }
          app_manager = AppManagerClient.new(my_node.private_ip)
          Djinn.log_info("Restarting AppServers hosting old version of #{app_name}.")
          begin
            result = app_manager.restart_app_instances_for_app(app_name, app_language)
          rescue FailedNodeException
            Djinn.log_warn("Failed to talk to app_manager to restart #{app_name}.")
            result = false
          end
          unless result
            Djinn.log_warn("Failed to restart app #{app_name}.")
          end
          maybe_reload_taskqueue_worker(app_name)
          APPS_LOCK.synchronize {
            @apps_to_restart.delete(app_name)
          }
          Djinn.log_info("Done restarting #{app_name}.")
        }
        if !no_appservers[0].nil?
          app = no_appservers[0]
          Djinn.log_info("Starting first AppServer for app: #{app}.")
          ret = add_appserver_process(app, @app_info_map[app]['nginx'],
            @app_info_map[app]['nginx_https'], @app_info_map[app]['language'])
          Djinn.log_debug("add_appserver_process returned: #{ret}.")
        elsif !to_start[0].nil?
          app = to_start[0]
          Djinn.log_info("Starting AppServer for app: #{app}.")
          ret = add_appserver_process(app, @app_info_map[app]['nginx'],
            @app_info_map[app]['nginx_https'], @app_info_map[app]['language'])
          Djinn.log_debug("add_appserver_process returned: #{ret}.")
        elsif !to_end[0].nil?
          Djinn.log_info("Terminate the following AppServer: #{to_end[0]}.")
          app, port = to_end[0].split(":")
          ret = remove_appserver_process(app, port)
          Djinn.log_debug("remove_appserver_process returned: #{ret}.")
        end
      }
    }
  end
def check_running_apps ( ) if AMS_LOCK . locked? Djinn . log_debug ( "Another thread already working with appmanager." ) return end CronHelper . clear_app_crontabs ( ) unless my_node . is_shadow? to_start = [ ] no_appservers = [ ] my_apps = [ ] to_end = [ ] APPS_LOCK . synchronize { @app_info_map . each { | app , info | next unless info [ 'appengine' ] if my_node . is_shadow? if ! info [ 'nginx' ] . nil? and ! info [ 'language' ] . nil? CronHelper . update_cron ( get_load_balancer_ip ( ) , info [ 'nginx' ] , info [ 'language' ] , app ) end end Djinn . log_debug ( "Checking #{app} with appengine #{info}." ) info [ 'appengine' ] . each { | location | host , port = location . split ( ":" ) next if @my_private_ip != host if Integer ( port ) < 0 to_start << app no_appservers << app elsif not MonitInterface . is_running? ( "#{app}-#{port}" ) Djinn . log_warn ( "Didn't find the AppServer for #{app} at port #{port}." ) to_end << "#{app}:#{port}" else my_apps << "#{app}:#{port}" end } } } my_apps . each { | appserver | app , _ = appserver . split ( ":" ) no_appservers . delete ( app ) } Djinn . log_debug ( "Running AppServers on this node: #{my_apps}." ) unless my_apps . empty? MonitInterface . running_appengines ( ) . each { | appengine | next if my_apps . include? ( appengine ) app , _ = appengine . split ( ":" ) time_to_delete = true if @last_decision [ app ] if Time . now . to_i - @last_decision [ app ] < APP_UPLOAD_TIMEOUT * RETRIES time_to_delete = false end end if to_start . include? ( app ) and ! time_to_delete Djinn . log_debug ( "Ignoring request for #{app} since we have pending AppServers." ) to_start . delete ( app ) no_appservers . delete ( app ) else to_end << appengine end } Djinn . log_debug ( "First AppServers to start: #{no_appservers}." ) unless no_appservers . empty? Djinn . log_debug ( "AppServers to start: #{to_start}." ) unless to_start . empty? Djinn . log_debug ( "AppServers to terminate: #{to_end}." ) unless to_end . empty? Thread . new { AMS_LOCK . synchronize { Djinn . log_debug ( "Checking if any app need to be restarted." ) apps_to_restart = [ ] app_language = "" APPS_LOCK . synchronize { apps_to_restart = @apps_to_restart } apps_to_restart . each { | app_name | Djinn . log_info ( "Got #{app_name} to restart (if applicable)." ) APPS_LOCK . synchronize { next unless @apps_to_restart . include? ( app_name ) setup_appengine_application ( app_name ) app_language = @app_info_map [ app_name ] [ 'language' ] } app_manager = AppManagerClient . new ( my_node . private_ip ) Djinn . log_info ( "Restarting AppServers hosting old version of #{app_name}." ) begin result = app_manager . restart_app_instances_for_app ( app_name , app_language ) rescue FailedNodeException Djinn . log_warn ( "Failed to talk to app_manager to restart #{app_name}." ) result = false end unless result Djinn . log_warn ( "Failed to restart app #{app_name}." ) end maybe_reload_taskqueue_worker ( app_name ) APPS_LOCK . synchronize { @apps_to_restart . delete ( app_name ) } Djinn . log_info ( "Done restarting #{app_name}." ) } if ! no_appservers [ 0 ] . nil? app = no_appservers [ 0 ] Djinn . log_info ( "Starting first AppServer for app: #{app}." ) ret = add_appserver_process ( app , @app_info_map [ app ] [ 'nginx' ] , @app_info_map [ app ] [ 'nginx_https' ] , @app_info_map [ app ] [ 'language' ] ) Djinn . log_debug ( "add_appserver_process returned: #{ret}." ) elsif ! to_start [ 0 ] . nil? app = to_start [ 0 ] Djinn . log_info ( "Starting AppServer for app: #{app}." ) ret = add_appserver_process ( app , @app_info_map [ app ] [ 'nginx' ] , @app_info_map [ app ] [ 'nginx_https' ] , @app_info_map [ app ] [ 'language' ] ) Djinn . log_debug ( "add_appserver_process returned: #{ret}." ) elsif ! to_end [ 0 ] . nil? Djinn . log_info ( "Terminate the following AppServer: #{to_end[0]}." ) app , port = to_end [ 0 ] . split ( ":" ) ret = remove_appserver_process ( app , port ) Djinn . log_debug ( "remove_appserver_process returned: #{ret}." ) end } } end
All nodes will compare the list of AppServers they should be running,
with the list of AppServers actually running, and make the necessary
adjustments.
All nodes will compare the list of AppServers they should be running with the list of AppServers actually running and make the necessary adjustments .
# The running AppServers on this node must match the login node view. # Only one thread talking to the AppManagerServer at a time. # Only the shadow runs the application cron jobs. # Update the cron job for the app on the shadow. # Let's make sure we have the proper list of apps with no currently # running AppServers. # Check that all the AppServers running are indeed known to the # head node. # Nothing to do if we already account for this AppServer. # If the app needs to be started, but we have an AppServer not # accounted for, we don't take action (ie we wait for headnode # state to settle). # Now we do the talking with the appmanagerserver. Since it may take # some time to start/stop apps, we do this in a thread. We do one # operation at a time since it is expensive and we want to # re-evaluate. # We restart applications first if needed. # Since the AMS can take a long time and we don't hold the # lock here on @apps_to_restart, we do a quick check before # doing the work. # TODO: What happens if the user updates their env vars between app # deploys? # We then start or terminate AppServers as needed.