title: nginx upstream connection timed out with AWS ELB
body: <p>We have nginx webservers which serves as reverse proxy and forwards request to AWS ELB. AWS ELB then forwards request to set of back-end servers which respond to requests. We used it successfully for last couple of years with nginx proxy_pass directive with nginx version 1.8. We recently installed a new cluster where nginx version is 1.11.9 and can see upstream connection time out errors in logs after couple of days.I can understand that it is happening when AWS ELB is changing its IP but don't understand why nginx is caching IP.</p>
  
  <p>Logs:</p>
  
  <pre><code>2017/03/07 06:12:01 [warn] 5322#5322: *41004 upstream server temporarily         disabled while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET/TT11617689 HTTP/1.1", upstream: "http://172.21.107.90:80/TT11617689", host: "localhost:8008"
  </code></pre>
  
  <p>2017/03/07 06:12:01 [error] 5322#5322: *41004 upstream timed out (110: Connection timed out) while connecting to upstream, client: 127.0.0.1</p>
  
  <p><em>Nginx configuration</em></p>
  
  <pre><code>user  nginx;
  worker_processes  1;
  
  error_log  /var/log/nginx/error.log warn;
  pid        /var/run/nginx.pid;
  
  
  events {
  worker_connections  1024;
  }
  
  
  http {
  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;
  
  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
  
  access_log  /var/log/nginx/access.log  main;
  
  sendfile        on;
  #tcp_nopush     on;
  
  keepalive_timeout  65;
  
  #gzip  on;
  
  include /etc/nginx/conf.d/*.conf;
  }
  
  server {
  listen       8008;
  server_name  localhost;
  set $elb "&lt;&lt;my elb url&gt;&gt;";
  #charset koi8-r;
  #access_log  /var/log/nginx/log/host.access.log  main;
  
   if ($request_method !~ ^(GET|POST)$ )
    {
        return 405;
   }
  
   location / {
      resolver 172.16.0.2 valid=10s;
      proxy_set_header Host $host;
      proxy_set_header X-Forwarded-Host $host;
      proxy_set_header X-Forwarded-Server $host;
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_pass $elb;
  
    }
  
    location /login {
     deny all;
     return 404;
   }
  
  
  
  
  error_page   500 502 503 504  /50x.html;
  location = /50x.html {
      root   /usr/share/nginx/html;
  }
  }
  </code></pre>