title: Nginx Chunked Encoding Response Randomly Truncated
body: <p>I have an upstream server that simply starts a chunked encoding response and sends a small chunk every second for 100 chunks.</p>
  
  <p>When I hit the upstream server directly like so, all works fine.</p>
  
  <pre><code>curl --raw 10.244.7.248:4000/test/stream
  </code></pre>
  
  <p>However, when I go through nginx like so, the response gets cut off at random times.</p>
  
  <pre><code>curl --raw localhost/test/stream
  </code></pre>
  
  <p>Curl returns</p>
  
  <pre><code>curl: (18) transfer closed with outstanding read data remaining
  </code></pre>
  
  <p>and the nginx logs only show</p>
  
  <pre><code>::1 - [::1] - - [24/Oct/2017:11:09:02 +0000] "GET /test/stream HTTP/1.1" 200 440 "-" "curl/7.47.0" 95 44.783 [bar-bar-80] 10.244.7.248:4000 440 44.783 200
  </code></pre>
  
  <p>Notice in this example, the response appears to complete successfully with a 200 but only after 44.783 seconds when it should have been roughly 100 seconds. Every time I try this I get a cut off at a different number of seconds.</p>
  
  <p>I am running Kubernetes Nginx Ingress Controller 0.9.0-beta.14 with <code>gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.14</code></p>
  
  <p>The nginx.conf is very long, but here are some snippets</p>
  
  <pre><code>daemon off;
  
  worker_processes 2;
  pid /run/nginx.pid;
  
  worker_rlimit_nofile 498976;
  worker_shutdown_timeout 10s ;
  
  events {
      multi_accept        on;
      worker_connections  16384;
      use                 epoll;
  }
  
  http {
      real_ip_header      X-Forwarded-For;
  
      real_ip_recursive   on;
      set_real_ip_from    0.0.0.0/0;
  
      geoip_country       /etc/nginx/GeoIP.dat;
      geoip_city          /etc/nginx/GeoLiteCity.dat;
      geoip_proxy_recursive on;
      sendfile            on;
  
      aio                 threads;
      aio_write           on;
  
      tcp_nopush          on;
      tcp_nodelay         on;
  
      log_subrequest      on;
  
      reset_timedout_connection on;
  
      keepalive_timeout  75s;
      keepalive_requests 100;
  
      client_header_buffer_size       1k;
      client_header_timeout           60s;
      large_client_header_buffers     4 8k;
      client_body_buffer_size         8k;
      client_body_timeout             60s;
  
      http2_max_field_size            4k;
      http2_max_header_size           16k;
  
      types_hash_max_size             2048;
      server_names_hash_max_size      8192;
      server_names_hash_bucket_size   128;
      map_hash_bucket_size            64;
  
      proxy_headers_hash_max_size     512;
      proxy_headers_hash_bucket_size  64;
  
      variables_hash_bucket_size      64;
      variables_hash_max_size         2048;
  
      underscores_in_headers          off;
      ignore_invalid_headers          on;
      include /etc/nginx/mime.types;
      default_type text/html;
      gzip on;
      gzip_comp_level 5;
      gzip_http_version 1.1;
      gzip_min_length 256;
      gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest
  +json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component;
      gzip_proxied any;
  
      # Custom headers for response
  
      server_tokens on;
  
      # disable warnings
      uninitialized_variable_warn off;
  
      # Additional available variables:
      # $namespace
      # $ingress_name
      # $service_name
      log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstrea
  m_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';
  
      map $request_uri $loggable {
          default 1;
      }
  
      access_log /var/log/nginx/access.log upstreaminfo if=$loggable;
      error_log  /var/log/nginx/error.log notice;
  
      resolver 10.0.0.10 valid=30s;
  
      # Retain the default nginx handling of requests without a "Connection" header
      map $http_upgrade $connection_upgrade {
          default          upgrade;
          ''               close;
      }
  
      # Trust HTTP X-Forwarded-* Headers, but use direct values if they're missing.
      map $http_x_forwarded_for $the_real_ip {
          # Get IP address from X-Forwarded-For HTTP header
          default          $realip_remote_addr;
          ''               $remote_addr;
      }
  
      # trust http_x_forwarded_proto headers correctly indicate ssl offloading
      map $http_x_forwarded_proto $pass_access_scheme {
          default          $http_x_forwarded_proto;
          ''               $scheme;
      }
  
      map $http_x_forwarded_port $pass_server_port {
          default           $http_x_forwarded_port;
          ''                $server_port;
      }
      map $http_x_forwarded_host $best_http_host {
          default          $http_x_forwarded_host;
          ''               $this_host;
      }
      map $pass_server_port $pass_port {
          443              443;
          default          $pass_server_port;
      }
  
      # Map a response error watching the header Content-Type
      map $http_accept $httpAccept {
          default          html;
          application/json json;
          application/xml  xml;
          text/plain       text;
      }
  
      map $httpAccept $httpReturnType {
          default          text/html;
          json             application/json;
          xml              application/xml;
          text             text/plain;
      }
  
      # Obtain best http host
      map $http_host $this_host {
          default          $http_host;
          ''               $host;
      }
  
      server_name_in_redirect off;
      port_in_redirect        off;
  
      ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
  
      # turn on session caching to drastically improve performance
      ssl_session_cache builtin:1000 shared:SSL:10m;
      ssl_session_timeout 10m;
  
      # allow configuring ssl session tickets
      ssl_session_tickets on;
  
      # slightly reduce the time-to-first-byte
      ssl_buffer_size 4k;
  
      # allow configuring custom ssl ciphers
      ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-R
  SA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:D
  HE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-C
  BC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
      ssl_prefer_server_ciphers on;
  
      ssl_ecdh_curve auto;
  
      proxy_ssl_session_reuse on;
      upstream bar-bar-80 {
          # Load balance algorithm; empty for round robin, which is the default
          least_conn;
  
          keepalive 32;
  
          server 10.244.7.248:4000 max_fails=0 fail_timeout=0;
      }
      server {
          server_name bar.gigalixir.com;
          listen 80;
          listen [::]:80;
          set $proxy_upstream_name "-";
  
          listen 443  ssl http2;
          listen [::]:443  ssl http2;
          ssl_certificate                         /ingress-controller/ssl/bar-bar-tls.pem;
          ssl_certificate_key                     /ingress-controller/ssl/bar-bar-tls.pem;
  
          more_set_headers                        "Strict-Transport-Security: max-age=15724800; includeSubDomains;";
          location / {
              set $proxy_upstream_name "bar-bar-80";
  
              set $namespace      "bar";
              set $ingress_name   "bar";
              set $service_name   "bar";
  
              # enforce ssl on server side
              if ($pass_access_scheme = http) {
                  return 301 https://$best_http_host$request_uri;
              }
              port_in_redirect off;
  
              client_max_body_size                    "0";
  
              proxy_set_header Host                   $best_http_host;
              # Pass the extracted client certificate to the backend
  
              # Allow websocket connections
              proxy_set_header                        Upgrade           $http_upgrade;
              proxy_set_header                        Connection        $connection_upgrade;
  
              proxy_set_header X-Real-IP              $the_real_ip;
              proxy_set_header X-Forwarded-For        $the_real_ip;
              proxy_set_header X-Forwarded-Host       $best_http_host;
              proxy_set_header X-Forwarded-Port       $pass_port;
              proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
              proxy_set_header X-Original-URI         $request_uri;
              proxy_set_header X-Scheme               $pass_access_scheme;
  
              proxy_set_header X-Auth-Request-Redirect $request_uri;
              # mitigate HTTPoxy Vulnerability
              # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
              proxy_set_header Proxy                  "";
  
              # Custom headers to proxied server
  
              proxy_connect_timeout                   5s;
              proxy_send_timeout                      60s;
              proxy_read_timeout                      60s;
  
              proxy_redirect                          off;
              proxy_buffering                         off;
              proxy_buffer_size                       "4k";
              proxy_buffers                           4 "4k";
              proxy_request_buffering                 "on";
  
              proxy_http_version                      1.1;
  
              proxy_cookie_domain                     off;
              proxy_cookie_path                       off;
  
              # In case of errors try the next upstream server before returning an error
              proxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;
  
              proxy_pass http://bar-bar-80;
          }
      }
  </code></pre>