Nginx struggling to accept connections during peak load

Hello,
I had posted to the mailing list earlier this week, but I managed to
gather
some new information that points directly to nginx (almost certainly my
configuration), so I thought I’d post something more concise.

I am running edge boxes which use nginx to terminate SSL which passes to
haproxy on the same server. During our peak load time, we are
experiencing
intermittent slow connection issues which drives up our response time
graphs
from external sources. Every log within our infrastructure shows no
problems, including the edge nginx that we’re having issues with.

Today, I was able to setup some boxes from different providers and run
some
curl tests in a loop. I setup a bash script that made a curl request to
our
edge nginx server for a specific API call. In another bash script, I
made a
curl request for the same API call, but bypassing nginx and going
directly
to haproxy that is located on the same exact box. By doing this, the
curls
to the nginx server showed intermittent big delays in the connection
phase
before nginx picks up the phone. The haproxy logs showed absolutely no
issues at all in connecting. Because haproxy is on the same server, I
believe that rules out anything related to a networking issue, both
physical
and kernel related.

My SSL connections usually look like this from a cURL:

        time_namelookup:  0.001
           time_connect:  0.035
        time_appconnect:  0.109
       time_pretransfer:  0.109
          time_redirect:  0.000
     time_starttransfer:  0.150
                        ----------
             time_total:  0.150

During my peak load, they intermittently (every 3-5 seconds) look like
this
(though most of the time, 3 seconds)

        time_namelookup:  0.001
           time_connect:  9.033
        time_appconnect:  9.109
       time_pretransfer:  9.109
          time_redirect:  0.000
     time_starttransfer:  9.148
                        ----------
             time_total:  9.148

So, here is my nginx config. I’m running nginx 1.4.1. The system itself
doesn’t go beyond 30% CPU combined and all other metrics look good as
well.
What can I do better (I’m sure lots)?

user www-data;
worker_processes 11; # 12 cores, 24 with HT
worker_rlimit_nofile 500000;
error_log /var/log/nginx/error.log;
pid /var/run/nginx.pid;

events {
use epoll;
multi_accept off;
accept_mutex off;
worker_connections 65536;
}

http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
proxy_buffering off;

log_format access '$http_x_forwarded_for - $remote_user [$time_local]

'"$request" $status $body_bytes_sent ’
‘"$http_referer" “$http_user_agent” “$host”
“$request_time”
“$upstream_response_time”’;

upstream apiv2-ssl {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}

upstream api {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}

upstream secure {
server 127.0.0.1:xxxxxx max_fails=3 fail_timeout=15s;
}

upstream facebook {
server 127.0.0.1:xxxxx max_fails=3 fail_timeout=15s;
}

upstream testing {
server 127.0.0.1:xxxxx max_fails=3 fail_timeout=15s;
}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv2.log access;
error_log /var/log/nginx/error_apiv2.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxxx.key;
ssl_session_cache shared:SSLv2:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://apiv2-ssl;
proxy_http_version 1.1;
proxy_set_header Connection “”;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;

ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv3.log access;
error_log /var/log/nginx/error_apiv3.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
  proxy_pass http://api;
  proxy_http_version 1.1;
  proxy_set_header Connection "";
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header Host $http_host;
  set $msecstart "${msec}000";
  if ($msecstart ~ "^(.*)\.(.*)") {set $msecout "t=$1$2";}
  proxy_set_header X-Request-Start $msecout;
}

}

server {
listen x.x.x.x:443;

ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_apiv3.log access;
error_log /var/log/nginx/error_apiv3.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
  proxy_pass http://testing;
  proxy_http_version 1.1;
  proxy_set_header Connection "";
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header Host $http_host;
}

}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_secure.log access;
error_log /var/log/nginx/error_secure.log;
gzip on;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxxx.key;
ssl_session_cache shared:SSLsecure:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://secure;
proxy_http_version 1.1;
proxy_set_header Connection “”;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_facebook.log access;
error_log /var/log/nginx/error_facebook.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/xxx.key;
ssl_session_cache shared:SSLfacebook:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://facebook;
proxy_http_version 1.1;
proxy_set_header Connection “”;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;
ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access_api.log access;
error_log /var/log/nginx/error_api.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLapi:500m;
ssl_ciphers ALL:!kEDH;
location / {
proxy_pass http://api;
proxy_http_version 1.1;
proxy_set_header Connection “”;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}

server {
listen x.x.x.x:443;
listen x.x.x.x:443;

ssl on;
keepalive_timeout 5 5;
access_log /var/log/nginx/access.log access;
error_log /var/log/nginx/error.log;
ssl_certificate /etc/nginx/certs/xxx.crt;
ssl_certificate_key /etc/nginx/certs/xxx.key;
ssl_session_cache shared:SSLv3:500m;
ssl_ciphers ALL:!kEDH;
location / {
  proxy_pass http://facebook;
  proxy_http_version 1.1;
  proxy_set_header Connection "";
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header Host $http_host;
}

}

}

Posted at Nginx Forum:
http://forum.nginx.org/read.php?2,243245,243245#msg-243245

This forum is not affiliated to the Ruby language, Ruby on Rails framework, nor any Ruby applications discussed here.

| Privacy Policy | Terms of Service | Remote Ruby Jobs